VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 72747

Last change on this file since 72747 was 72747, checked in by vboxsync, 6 years ago

VMM: Extend HM changed flags. ​​​bugref:9193 [build fix]
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 573.0 KB
Line 
1/* $Id: HMVMXR0.cpp 72747 2018-06-29 07:48:28Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/x86.h>
25#include <iprt/asm-amd64-x86.h>
26#include <iprt/thread.h>
27
28#include <VBox/vmm/pdmapi.h>
29#include <VBox/vmm/dbgf.h>
30#include <VBox/vmm/iem.h>
31#include <VBox/vmm/iom.h>
32#include <VBox/vmm/selm.h>
33#include <VBox/vmm/tm.h>
34#include <VBox/vmm/gim.h>
35#include <VBox/vmm/apic.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#include "HMInternal.h"
40#include <VBox/vmm/vm.h>
41#include "HMVMXR0.h"
42#include "dtrace/VBoxVMM.h"
43
44#ifdef DEBUG_ramshankar
45# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50# define HMVMX_ALWAYS_TRAP_PF
51# define HMVMX_ALWAYS_FLUSH_TLB
52# define HMVMX_ALWAYS_SWAP_EFER
53#endif
54
55
56/*********************************************************************************************************************************
57* Defined Constants And Macros *
58*********************************************************************************************************************************/
59/** Use the function table. */
60#define HMVMX_USE_FUNCTION_TABLE
61
62/** Determine which tagged-TLB flush handler to use. */
63#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68/** @name HMVMX_READ_XXX
69 * Flags to skip redundant reads of some common VMCS fields that are not part of
70 * the guest-CPU or VCPU state but are needed while handling VM-exits.
71 */
72#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
73#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
74#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
75#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
76#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
77#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
78#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
79/** @} */
80
81/**
82 * States of the VMCS.
83 *
84 * This does not reflect all possible VMCS states but currently only those
85 * needed for maintaining the VMCS consistently even when thread-context hooks
86 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
87 */
88#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
89#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
90#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
91
92/**
93 * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
94 * guest using hardware-assisted VMX.
95 *
96 * This excludes state like GPRs (other than RSP) which are always are
97 * swapped and restored across the world-switch and also registers like EFER,
98 * MSR which cannot be modified by the guest without causing a VM-exit.
99 */
100#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
101 | CPUMCTX_EXTRN_RFLAGS \
102 | CPUMCTX_EXTRN_RSP \
103 | CPUMCTX_EXTRN_SREG_MASK \
104 | CPUMCTX_EXTRN_TABLE_MASK \
105 | CPUMCTX_EXTRN_KERNEL_GS_BASE \
106 | CPUMCTX_EXTRN_SYSCALL_MSRS \
107 | CPUMCTX_EXTRN_SYSENTER_MSRS \
108 | CPUMCTX_EXTRN_TSC_AUX \
109 | CPUMCTX_EXTRN_OTHER_MSRS \
110 | CPUMCTX_EXTRN_CR0 \
111 | CPUMCTX_EXTRN_CR3 \
112 | CPUMCTX_EXTRN_CR4 \
113 | CPUMCTX_EXTRN_DR7 \
114 | CPUMCTX_EXTRN_HM_VMX_MASK)
115
116/**
117 * Exception bitmap mask for real-mode guests (real-on-v86).
118 *
119 * We need to intercept all exceptions manually except:
120 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
121 * due to bugs in Intel CPUs.
122 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
123 * support.
124 */
125#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
126 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
127 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
128 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
129 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
130 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
131 | RT_BIT(X86_XCPT_XF))
132
133/**
134 * Exception bitmap mask for all contributory exceptions.
135 *
136 * Page fault is deliberately excluded here as it's conditional as to whether
137 * it's contributory or benign. Page faults are handled separately.
138 */
139#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
140 | RT_BIT(X86_XCPT_DE))
141
142/** Maximum VM-instruction error number. */
143#define HMVMX_INSTR_ERROR_MAX 28
144
145/** Profiling macro. */
146#ifdef HM_PROFILE_EXIT_DISPATCH
147# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
148# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
149#else
150# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
151# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
152#endif
153
154/** Assert that preemption is disabled or covered by thread-context hooks. */
155#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
156 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
157
158/** Assert that we haven't migrated CPUs when thread-context hooks are not
159 * used. */
160#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
161 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
162 ("Illegal migration! Entered on CPU %u Current %u\n", \
163 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
164
165/** Helper macro for VM-exit handlers called unexpectedly. */
166#define HMVMX_RETURN_UNEXPECTED_EXIT() \
167 do { \
168 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
169 return VERR_VMX_UNEXPECTED_EXIT; \
170 } while (0)
171
172/** Macro for saving segment registers from VMCS into the guest-CPU
173 * context. */
174#ifdef VMX_USE_CACHED_VMCS_ACCESSES
175# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
176 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
177 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
178#else
179# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
180 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
181 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
182#endif
183
184
185/*********************************************************************************************************************************
186* Structures and Typedefs *
187*********************************************************************************************************************************/
188/**
189 * VMX transient state.
190 *
191 * A state structure for holding miscellaneous information across
192 * VMX non-root operation and restored after the transition.
193 */
194typedef struct VMXTRANSIENT
195{
196 /** The host's rflags/eflags. */
197 RTCCUINTREG fEFlags;
198#if HC_ARCH_BITS == 32
199 uint32_t u32Alignment0;
200#endif
201 /** The guest's TPR value used for TPR shadowing. */
202 uint8_t u8GuestTpr;
203 /** Alignment. */
204 uint8_t abAlignment0[7];
205
206 /** The basic VM-exit reason. */
207 uint16_t uExitReason;
208 /** Alignment. */
209 uint16_t u16Alignment0;
210 /** The VM-exit interruption error code. */
211 uint32_t uExitIntErrorCode;
212 /** The VM-exit exit code qualification. */
213 uint64_t uExitQualification;
214
215 /** The VM-exit interruption-information field. */
216 uint32_t uExitIntInfo;
217 /** The VM-exit instruction-length field. */
218 uint32_t cbInstr;
219 /** The VM-exit instruction-information field. */
220 union
221 {
222 /** Plain unsigned int representation. */
223 uint32_t u;
224 /** INS and OUTS information. */
225 struct
226 {
227 uint32_t u7Reserved0 : 7;
228 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
229 uint32_t u3AddrSize : 3;
230 uint32_t u5Reserved1 : 5;
231 /** The segment register (X86_SREG_XXX). */
232 uint32_t iSegReg : 3;
233 uint32_t uReserved2 : 14;
234 } StrIo;
235 /** INVEPT, INVVPID, INVPCID information. */
236 struct
237 {
238 /** Scaling; 0=no scaling, 1=scale-by-2, 2=scale-by-4, 3=scale-by-8. */
239 uint32_t u2Scaling : 2;
240 uint32_t u5Reserved0 : 5;
241 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
242 uint32_t u3AddrSize : 3;
243 uint32_t u1Reserved0 : 1;
244 uint32_t u4Reserved0 : 4;
245 /** The segment register (X86_SREG_XXX). */
246 uint32_t iSegReg : 3;
247 /** The index register (X86_GREG_XXX). */
248 uint32_t iIdxReg : 4;
249 /** Set if index register is invalid. */
250 uint32_t fIdxRegValid : 1;
251 /** The base register (X86_GREG_XXX). */
252 uint32_t iBaseReg : 4;
253 /** Set if base register is invalid. */
254 uint32_t fBaseRegValid : 1;
255 /** Register 2 (X86_GREG_XXX). */
256 uint32_t iReg2 : 4;
257 } Inv;
258 } ExitInstrInfo;
259 /** Whether the VM-entry failed or not. */
260 bool fVMEntryFailed;
261 /** Alignment. */
262 uint8_t abAlignment1[3];
263
264 /** The VM-entry interruption-information field. */
265 uint32_t uEntryIntInfo;
266 /** The VM-entry exception error code field. */
267 uint32_t uEntryXcptErrorCode;
268 /** The VM-entry instruction length field. */
269 uint32_t cbEntryInstr;
270
271 /** IDT-vectoring information field. */
272 uint32_t uIdtVectoringInfo;
273 /** IDT-vectoring error code. */
274 uint32_t uIdtVectoringErrorCode;
275
276 /** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
277 uint32_t fVmcsFieldsRead;
278
279 /** Whether the guest debug state was active at the time of VM-exit. */
280 bool fWasGuestDebugStateActive;
281 /** Whether the hyper debug state was active at the time of VM-exit. */
282 bool fWasHyperDebugStateActive;
283 /** Whether TSC-offsetting should be setup before VM-entry. */
284 bool fUpdateTscOffsettingAndPreemptTimer;
285 /** Whether the VM-exit was caused by a page-fault during delivery of a
286 * contributory exception or a page-fault. */
287 bool fVectoringDoublePF;
288 /** Whether the VM-exit was caused by a page-fault during delivery of an
289 * external interrupt or NMI. */
290 bool fVectoringPF;
291} VMXTRANSIENT;
292AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
293AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
294AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
295AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
296AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
297/** Pointer to VMX transient state. */
298typedef VMXTRANSIENT *PVMXTRANSIENT;
299
300
301/**
302 * MSR-bitmap read permissions.
303 */
304typedef enum VMXMSREXITREAD
305{
306 /** Reading this MSR causes a VM-exit. */
307 VMXMSREXIT_INTERCEPT_READ = 0xb,
308 /** Reading this MSR does not cause a VM-exit. */
309 VMXMSREXIT_PASSTHRU_READ
310} VMXMSREXITREAD;
311/** Pointer to MSR-bitmap read permissions. */
312typedef VMXMSREXITREAD* PVMXMSREXITREAD;
313
314/**
315 * MSR-bitmap write permissions.
316 */
317typedef enum VMXMSREXITWRITE
318{
319 /** Writing to this MSR causes a VM-exit. */
320 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
321 /** Writing to this MSR does not cause a VM-exit. */
322 VMXMSREXIT_PASSTHRU_WRITE
323} VMXMSREXITWRITE;
324/** Pointer to MSR-bitmap write permissions. */
325typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
326
327
328/**
329 * VMX VM-exit handler.
330 *
331 * @returns Strict VBox status code (i.e. informational status codes too).
332 * @param pVCpu The cross context virtual CPU structure.
333 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
334 * out-of-sync. Make sure to update the required
335 * fields before using them.
336 * @param pVmxTransient Pointer to the VMX-transient structure.
337 */
338#ifndef HMVMX_USE_FUNCTION_TABLE
339typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
340#else
341typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
342/** Pointer to VM-exit handler. */
343typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
344#endif
345
346/**
347 * VMX VM-exit handler, non-strict status code.
348 *
349 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
350 *
351 * @returns VBox status code, no informational status code returned.
352 * @param pVCpu The cross context virtual CPU structure.
353 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
354 * out-of-sync. Make sure to update the required
355 * fields before using them.
356 * @param pVmxTransient Pointer to the VMX-transient structure.
357 *
358 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
359 * use of that status code will be replaced with VINF_EM_SOMETHING
360 * later when switching over to IEM.
361 */
362#ifndef HMVMX_USE_FUNCTION_TABLE
363typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
364#else
365typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
366#endif
367
368
369/*********************************************************************************************************************************
370* Internal Functions *
371*********************************************************************************************************************************/
372static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
373static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
374static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
375static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
376static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
377 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
378#if HC_ARCH_BITS == 32
379static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
380#endif
381#ifndef HMVMX_USE_FUNCTION_TABLE
382DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
383# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
384# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
385#else
386# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
387# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
388#endif
389
390
391/** @name VM-exit handlers.
392 * @{
393 */
394static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
395static FNVMXEXITHANDLER hmR0VmxExitExtInt;
396static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
397static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
398static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
399static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
400static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
401static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
402static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
403static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
404static FNVMXEXITHANDLER hmR0VmxExitCpuid;
405static FNVMXEXITHANDLER hmR0VmxExitGetsec;
406static FNVMXEXITHANDLER hmR0VmxExitHlt;
407static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
408static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
409static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
410static FNVMXEXITHANDLER hmR0VmxExitVmcall;
411static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
412static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
413static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
414static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
415static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
416static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
417static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
418static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
419static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
420static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
421static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
422static FNVMXEXITHANDLER hmR0VmxExitMwait;
423static FNVMXEXITHANDLER hmR0VmxExitMtf;
424static FNVMXEXITHANDLER hmR0VmxExitMonitor;
425static FNVMXEXITHANDLER hmR0VmxExitPause;
426static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
427static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
428static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
429static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
430static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
431static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
432static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
433static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
434static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
435static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
436static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
437static FNVMXEXITHANDLER hmR0VmxExitRdrand;
438static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
439/** @} */
440
441static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
442static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
443static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
444static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
445static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
446static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
447static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
448static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
449
450
451/*********************************************************************************************************************************
452* Global Variables *
453*********************************************************************************************************************************/
454#ifdef HMVMX_USE_FUNCTION_TABLE
455
456/**
457 * VMX_EXIT dispatch table.
458 */
459static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
460{
461 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
462 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
463 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
464 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
465 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
466 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
467 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
468 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
469 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
470 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
471 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
472 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
473 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
474 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
475 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
476 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
477 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
478 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
479 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
480 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
481 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
482 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
483 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
484 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
485 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
486 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
487 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
488 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
489 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
490 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
491 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
492 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
493 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
494 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
495 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
496 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
497 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
498 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
499 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
500 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
501 /* 40 UNDEFINED */ hmR0VmxExitPause,
502 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
503 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
504 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
505 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
506 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
507 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
508 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
509 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
510 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
511 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
512 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
513 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
514 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
515 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
516 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
517 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
518 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
519 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
520 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
521 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
522 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
523 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
524 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
525 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
526};
527#endif /* HMVMX_USE_FUNCTION_TABLE */
528
529#ifdef VBOX_STRICT
530static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
531{
532 /* 0 */ "(Not Used)",
533 /* 1 */ "VMCALL executed in VMX root operation.",
534 /* 2 */ "VMCLEAR with invalid physical address.",
535 /* 3 */ "VMCLEAR with VMXON pointer.",
536 /* 4 */ "VMLAUNCH with non-clear VMCS.",
537 /* 5 */ "VMRESUME with non-launched VMCS.",
538 /* 6 */ "VMRESUME after VMXOFF",
539 /* 7 */ "VM-entry with invalid control fields.",
540 /* 8 */ "VM-entry with invalid host state fields.",
541 /* 9 */ "VMPTRLD with invalid physical address.",
542 /* 10 */ "VMPTRLD with VMXON pointer.",
543 /* 11 */ "VMPTRLD with incorrect revision identifier.",
544 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
545 /* 13 */ "VMWRITE to read-only VMCS component.",
546 /* 14 */ "(Not Used)",
547 /* 15 */ "VMXON executed in VMX root operation.",
548 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
549 /* 17 */ "VM-entry with non-launched executing VMCS.",
550 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
551 /* 19 */ "VMCALL with non-clear VMCS.",
552 /* 20 */ "VMCALL with invalid VM-exit control fields.",
553 /* 21 */ "(Not Used)",
554 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
555 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
556 /* 24 */ "VMCALL with invalid SMM-monitor features.",
557 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
558 /* 26 */ "VM-entry with events blocked by MOV SS.",
559 /* 27 */ "(Not Used)",
560 /* 28 */ "Invalid operand to INVEPT/INVVPID."
561};
562#endif /* VBOX_STRICT */
563
564
565
566/**
567 * Updates the VM's last error record.
568 *
569 * If there was a VMX instruction error, reads the error data from the VMCS and
570 * updates VCPU's last error record as well.
571 *
572 * @param pVM The cross context VM structure.
573 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
574 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
575 * VERR_VMX_INVALID_VMCS_FIELD.
576 * @param rc The error code.
577 */
578static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
579{
580 AssertPtr(pVM);
581 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
582 || rc == VERR_VMX_UNABLE_TO_START_VM)
583 {
584 AssertPtrReturnVoid(pVCpu);
585 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
586 }
587 pVM->hm.s.lLastError = rc;
588}
589
590
591/**
592 * Reads the VM-entry interruption-information field from the VMCS into the VMX
593 * transient structure.
594 *
595 * @returns VBox status code.
596 * @param pVmxTransient Pointer to the VMX transient structure.
597 *
598 * @remarks No-long-jump zone!!!
599 */
600DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
601{
602 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
603 AssertRCReturn(rc, rc);
604 return VINF_SUCCESS;
605}
606
607#ifdef VBOX_STRICT
608/**
609 * Reads the VM-entry exception error code field from the VMCS into
610 * the VMX transient structure.
611 *
612 * @returns VBox status code.
613 * @param pVmxTransient Pointer to the VMX transient structure.
614 *
615 * @remarks No-long-jump zone!!!
616 */
617DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
618{
619 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
620 AssertRCReturn(rc, rc);
621 return VINF_SUCCESS;
622}
623
624
625/**
626 * Reads the VM-entry exception error code field from the VMCS into
627 * the VMX transient structure.
628 *
629 * @returns VBox status code.
630 * @param pVmxTransient Pointer to the VMX transient structure.
631 *
632 * @remarks No-long-jump zone!!!
633 */
634DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
635{
636 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
637 AssertRCReturn(rc, rc);
638 return VINF_SUCCESS;
639}
640#endif /* VBOX_STRICT */
641
642
643/**
644 * Reads the VM-exit interruption-information field from the VMCS into the VMX
645 * transient structure.
646 *
647 * @returns VBox status code.
648 * @param pVmxTransient Pointer to the VMX transient structure.
649 */
650DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
651{
652 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
653 {
654 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
655 AssertRCReturn(rc,rc);
656 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_INFO;
657 }
658 return VINF_SUCCESS;
659}
660
661
662/**
663 * Reads the VM-exit interruption error code from the VMCS into the VMX
664 * transient structure.
665 *
666 * @returns VBox status code.
667 * @param pVmxTransient Pointer to the VMX transient structure.
668 */
669DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
670{
671 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
672 {
673 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
674 AssertRCReturn(rc, rc);
675 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
676 }
677 return VINF_SUCCESS;
678}
679
680
681/**
682 * Reads the VM-exit instruction length field from the VMCS into the VMX
683 * transient structure.
684 *
685 * @returns VBox status code.
686 * @param pVmxTransient Pointer to the VMX transient structure.
687 */
688DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
689{
690 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
691 {
692 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
693 AssertRCReturn(rc, rc);
694 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_LEN;
695 }
696 return VINF_SUCCESS;
697}
698
699
700/**
701 * Reads the VM-exit instruction-information field from the VMCS into
702 * the VMX transient structure.
703 *
704 * @returns VBox status code.
705 * @param pVmxTransient Pointer to the VMX transient structure.
706 */
707DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
708{
709 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
710 {
711 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
712 AssertRCReturn(rc, rc);
713 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_INFO;
714 }
715 return VINF_SUCCESS;
716}
717
718
719/**
720 * Reads the exit code qualification from the VMCS into the VMX transient
721 * structure.
722 *
723 * @returns VBox status code.
724 * @param pVCpu The cross context virtual CPU structure of the
725 * calling EMT. (Required for the VMCS cache case.)
726 * @param pVmxTransient Pointer to the VMX transient structure.
727 */
728DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
729{
730 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
731 {
732 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
733 AssertRCReturn(rc, rc);
734 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION;
735 }
736 return VINF_SUCCESS;
737}
738
739
740/**
741 * Reads the IDT-vectoring information field from the VMCS into the VMX
742 * transient structure.
743 *
744 * @returns VBox status code.
745 * @param pVmxTransient Pointer to the VMX transient structure.
746 *
747 * @remarks No-long-jump zone!!!
748 */
749DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
750{
751 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
752 {
753 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
754 AssertRCReturn(rc, rc);
755 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_INFO;
756 }
757 return VINF_SUCCESS;
758}
759
760
761/**
762 * Reads the IDT-vectoring error code from the VMCS into the VMX
763 * transient structure.
764 *
765 * @returns VBox status code.
766 * @param pVmxTransient Pointer to the VMX transient structure.
767 */
768DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
769{
770 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
771 {
772 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
773 AssertRCReturn(rc, rc);
774 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
775 }
776 return VINF_SUCCESS;
777}
778
779
780/**
781 * Enters VMX root mode operation on the current CPU.
782 *
783 * @returns VBox status code.
784 * @param pVM The cross context VM structure. Can be
785 * NULL, after a resume.
786 * @param HCPhysCpuPage Physical address of the VMXON region.
787 * @param pvCpuPage Pointer to the VMXON region.
788 */
789static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
790{
791 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
792 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
793 Assert(pvCpuPage);
794 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
795
796 if (pVM)
797 {
798 /* Write the VMCS revision dword to the VMXON region. */
799 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
800 }
801
802 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
803 RTCCUINTREG fEFlags = ASMIntDisableFlags();
804
805 /* Enable the VMX bit in CR4 if necessary. */
806 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
807
808 /* Enter VMX root mode. */
809 int rc = VMXEnable(HCPhysCpuPage);
810 if (RT_FAILURE(rc))
811 {
812 if (!(uOldCr4 & X86_CR4_VMXE))
813 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
814
815 if (pVM)
816 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
817 }
818
819 /* Restore interrupts. */
820 ASMSetFlags(fEFlags);
821 return rc;
822}
823
824
825/**
826 * Exits VMX root mode operation on the current CPU.
827 *
828 * @returns VBox status code.
829 */
830static int hmR0VmxLeaveRootMode(void)
831{
832 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
833
834 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
835 RTCCUINTREG fEFlags = ASMIntDisableFlags();
836
837 /* If we're for some reason not in VMX root mode, then don't leave it. */
838 RTCCUINTREG uHostCR4 = ASMGetCR4();
839
840 int rc;
841 if (uHostCR4 & X86_CR4_VMXE)
842 {
843 /* Exit VMX root mode and clear the VMX bit in CR4. */
844 VMXDisable();
845 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
846 rc = VINF_SUCCESS;
847 }
848 else
849 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
850
851 /* Restore interrupts. */
852 ASMSetFlags(fEFlags);
853 return rc;
854}
855
856
857/**
858 * Allocates and maps one physically contiguous page. The allocated page is
859 * zero'd out. (Used by various VT-x structures).
860 *
861 * @returns IPRT status code.
862 * @param pMemObj Pointer to the ring-0 memory object.
863 * @param ppVirt Where to store the virtual address of the
864 * allocation.
865 * @param pHCPhys Where to store the physical address of the
866 * allocation.
867 */
868static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
869{
870 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
871 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
872 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
873
874 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
875 if (RT_FAILURE(rc))
876 return rc;
877 *ppVirt = RTR0MemObjAddress(*pMemObj);
878 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
879 ASMMemZero32(*ppVirt, PAGE_SIZE);
880 return VINF_SUCCESS;
881}
882
883
884/**
885 * Frees and unmaps an allocated physical page.
886 *
887 * @param pMemObj Pointer to the ring-0 memory object.
888 * @param ppVirt Where to re-initialize the virtual address of
889 * allocation as 0.
890 * @param pHCPhys Where to re-initialize the physical address of the
891 * allocation as 0.
892 */
893static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
894{
895 AssertPtr(pMemObj);
896 AssertPtr(ppVirt);
897 AssertPtr(pHCPhys);
898 if (*pMemObj != NIL_RTR0MEMOBJ)
899 {
900 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
901 AssertRC(rc);
902 *pMemObj = NIL_RTR0MEMOBJ;
903 *ppVirt = 0;
904 *pHCPhys = 0;
905 }
906}
907
908
909/**
910 * Worker function to free VT-x related structures.
911 *
912 * @returns IPRT status code.
913 * @param pVM The cross context VM structure.
914 */
915static void hmR0VmxStructsFree(PVM pVM)
916{
917 for (VMCPUID i = 0; i < pVM->cCpus; i++)
918 {
919 PVMCPU pVCpu = &pVM->aCpus[i];
920 AssertPtr(pVCpu);
921
922 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
923 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
924
925 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
926 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
927
928 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
929 }
930
931 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
932#ifdef VBOX_WITH_CRASHDUMP_MAGIC
933 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
934#endif
935}
936
937
938/**
939 * Worker function to allocate VT-x related VM structures.
940 *
941 * @returns IPRT status code.
942 * @param pVM The cross context VM structure.
943 */
944static int hmR0VmxStructsAlloc(PVM pVM)
945{
946 /*
947 * Initialize members up-front so we can cleanup properly on allocation failure.
948 */
949#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
950 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
951 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
952 pVM->hm.s.vmx.HCPhys##a_Name = 0;
953
954#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
955 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
956 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
957 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
958
959#ifdef VBOX_WITH_CRASHDUMP_MAGIC
960 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
961#endif
962 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
963
964 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
965 for (VMCPUID i = 0; i < pVM->cCpus; i++)
966 {
967 PVMCPU pVCpu = &pVM->aCpus[i];
968 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
969 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
970 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
971 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
972 }
973#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
974#undef VMXLOCAL_INIT_VM_MEMOBJ
975
976 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
977 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
978 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
979 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
980
981 /*
982 * Allocate all the VT-x structures.
983 */
984 int rc = VINF_SUCCESS;
985#ifdef VBOX_WITH_CRASHDUMP_MAGIC
986 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
987 if (RT_FAILURE(rc))
988 goto cleanup;
989 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
990 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
991#endif
992
993 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
994 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
995 {
996 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
997 &pVM->hm.s.vmx.HCPhysApicAccess);
998 if (RT_FAILURE(rc))
999 goto cleanup;
1000 }
1001
1002 /*
1003 * Initialize per-VCPU VT-x structures.
1004 */
1005 for (VMCPUID i = 0; i < pVM->cCpus; i++)
1006 {
1007 PVMCPU pVCpu = &pVM->aCpus[i];
1008 AssertPtr(pVCpu);
1009
1010 /* Allocate the VM control structure (VMCS). */
1011 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1012 if (RT_FAILURE(rc))
1013 goto cleanup;
1014
1015 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1016 if ( PDMHasApic(pVM)
1017 && (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW))
1018 {
1019 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1020 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1021 if (RT_FAILURE(rc))
1022 goto cleanup;
1023 }
1024
1025 /*
1026 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1027 * transparent accesses of specific MSRs.
1028 *
1029 * If the condition for enabling MSR bitmaps changes here, don't forget to
1030 * update HMAreMsrBitmapsAvailable().
1031 */
1032 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1033 {
1034 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1035 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1036 if (RT_FAILURE(rc))
1037 goto cleanup;
1038 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1039 }
1040
1041 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1042 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1043 if (RT_FAILURE(rc))
1044 goto cleanup;
1045
1046 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1047 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1048 if (RT_FAILURE(rc))
1049 goto cleanup;
1050 }
1051
1052 return VINF_SUCCESS;
1053
1054cleanup:
1055 hmR0VmxStructsFree(pVM);
1056 return rc;
1057}
1058
1059
1060/**
1061 * Does global VT-x initialization (called during module initialization).
1062 *
1063 * @returns VBox status code.
1064 */
1065VMMR0DECL(int) VMXR0GlobalInit(void)
1066{
1067#ifdef HMVMX_USE_FUNCTION_TABLE
1068 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1069# ifdef VBOX_STRICT
1070 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1071 Assert(g_apfnVMExitHandlers[i]);
1072# endif
1073#endif
1074 return VINF_SUCCESS;
1075}
1076
1077
1078/**
1079 * Does global VT-x termination (called during module termination).
1080 */
1081VMMR0DECL(void) VMXR0GlobalTerm()
1082{
1083 /* Nothing to do currently. */
1084}
1085
1086
1087/**
1088 * Sets up and activates VT-x on the current CPU.
1089 *
1090 * @returns VBox status code.
1091 * @param pCpu Pointer to the global CPU info struct.
1092 * @param pVM The cross context VM structure. Can be
1093 * NULL after a host resume operation.
1094 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1095 * fEnabledByHost is @c true).
1096 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1097 * @a fEnabledByHost is @c true).
1098 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1099 * enable VT-x on the host.
1100 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1101 */
1102VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1103 void *pvMsrs)
1104{
1105 Assert(pCpu);
1106 Assert(pvMsrs);
1107 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1108
1109 /* Enable VT-x if it's not already enabled by the host. */
1110 if (!fEnabledByHost)
1111 {
1112 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1113 if (RT_FAILURE(rc))
1114 return rc;
1115 }
1116
1117 /*
1118 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1119 * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1120 * invalidated when flushing by VPID.
1121 */
1122 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1123 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1124 {
1125 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1126 pCpu->fFlushAsidBeforeUse = false;
1127 }
1128 else
1129 pCpu->fFlushAsidBeforeUse = true;
1130
1131 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1132 ++pCpu->cTlbFlushes;
1133
1134 return VINF_SUCCESS;
1135}
1136
1137
1138/**
1139 * Deactivates VT-x on the current CPU.
1140 *
1141 * @returns VBox status code.
1142 * @param pCpu Pointer to the global CPU info struct.
1143 * @param pvCpuPage Pointer to the VMXON region.
1144 * @param HCPhysCpuPage Physical address of the VMXON region.
1145 *
1146 * @remarks This function should never be called when SUPR0EnableVTx() or
1147 * similar was used to enable VT-x on the host.
1148 */
1149VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1150{
1151 NOREF(pCpu);
1152 NOREF(pvCpuPage);
1153 NOREF(HCPhysCpuPage);
1154
1155 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1156 return hmR0VmxLeaveRootMode();
1157}
1158
1159
1160/**
1161 * Sets the permission bits for the specified MSR in the MSR bitmap.
1162 *
1163 * @param pVCpu The cross context virtual CPU structure.
1164 * @param uMsr The MSR value.
1165 * @param enmRead Whether reading this MSR causes a VM-exit.
1166 * @param enmWrite Whether writing this MSR causes a VM-exit.
1167 */
1168static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1169{
1170 int32_t iBit;
1171 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1172
1173 /*
1174 * Layout:
1175 * 0x000 - 0x3ff - Low MSR read bits
1176 * 0x400 - 0x7ff - High MSR read bits
1177 * 0x800 - 0xbff - Low MSR write bits
1178 * 0xc00 - 0xfff - High MSR write bits
1179 */
1180 if (uMsr <= 0x00001FFF)
1181 iBit = uMsr;
1182 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1183 {
1184 iBit = uMsr - UINT32_C(0xC0000000);
1185 pbMsrBitmap += 0x400;
1186 }
1187 else
1188 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1189
1190 Assert(iBit <= 0x1fff);
1191 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1192 ASMBitSet(pbMsrBitmap, iBit);
1193 else
1194 ASMBitClear(pbMsrBitmap, iBit);
1195
1196 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1197 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1198 else
1199 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1200}
1201
1202
1203#ifdef VBOX_STRICT
1204/**
1205 * Gets the permission bits for the specified MSR in the MSR bitmap.
1206 *
1207 * @returns VBox status code.
1208 * @retval VINF_SUCCESS if the specified MSR is found.
1209 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1210 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1211 *
1212 * @param pVCpu The cross context virtual CPU structure.
1213 * @param uMsr The MSR.
1214 * @param penmRead Where to store the read permissions.
1215 * @param penmWrite Where to store the write permissions.
1216 */
1217static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1218{
1219 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1220 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1221 int32_t iBit;
1222 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1223
1224 /* See hmR0VmxSetMsrPermission() for the layout. */
1225 if (uMsr <= 0x00001FFF)
1226 iBit = uMsr;
1227 else if ( uMsr >= 0xC0000000
1228 && uMsr <= 0xC0001FFF)
1229 {
1230 iBit = (uMsr - 0xC0000000);
1231 pbMsrBitmap += 0x400;
1232 }
1233 else
1234 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1235
1236 Assert(iBit <= 0x1fff);
1237 if (ASMBitTest(pbMsrBitmap, iBit))
1238 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1239 else
1240 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1241
1242 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1243 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1244 else
1245 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1246 return VINF_SUCCESS;
1247}
1248#endif /* VBOX_STRICT */
1249
1250
1251/**
1252 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1253 * area.
1254 *
1255 * @returns VBox status code.
1256 * @param pVCpu The cross context virtual CPU structure.
1257 * @param cMsrs The number of MSRs.
1258 */
1259static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1260{
1261 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1262 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1263 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1264 {
1265 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1266 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1267 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1268 }
1269
1270 /* Update number of guest MSRs to load/store across the world-switch. */
1271 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1272 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1273
1274 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1275 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1276 AssertRCReturn(rc, rc);
1277
1278 /* Update the VCPU's copy of the MSR count. */
1279 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1280
1281 return VINF_SUCCESS;
1282}
1283
1284
1285/**
1286 * Adds a new (or updates the value of an existing) guest/host MSR
1287 * pair to be swapped during the world-switch as part of the
1288 * auto-load/store MSR area in the VMCS.
1289 *
1290 * @returns VBox status code.
1291 * @param pVCpu The cross context virtual CPU structure.
1292 * @param uMsr The MSR.
1293 * @param uGuestMsrValue Value of the guest MSR.
1294 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1295 * necessary.
1296 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1297 * its value was updated. Optional, can be NULL.
1298 */
1299static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1300 bool *pfAddedAndUpdated)
1301{
1302 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1303 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1304 uint32_t i;
1305 for (i = 0; i < cMsrs; i++)
1306 {
1307 if (pGuestMsr->u32Msr == uMsr)
1308 break;
1309 pGuestMsr++;
1310 }
1311
1312 bool fAdded = false;
1313 if (i == cMsrs)
1314 {
1315 ++cMsrs;
1316 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1317 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1318
1319 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1320 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1321 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1322
1323 fAdded = true;
1324 }
1325
1326 /* Update the MSR values in the auto-load/store MSR area. */
1327 pGuestMsr->u32Msr = uMsr;
1328 pGuestMsr->u64Value = uGuestMsrValue;
1329
1330 /* Create/update the MSR slot in the host MSR area. */
1331 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1332 pHostMsr += i;
1333 pHostMsr->u32Msr = uMsr;
1334
1335 /*
1336 * Update the host MSR only when requested by the caller AND when we're
1337 * adding it to the auto-load/store area. Otherwise, it would have been
1338 * updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1339 */
1340 bool fUpdatedMsrValue = false;
1341 if ( fAdded
1342 && fUpdateHostMsr)
1343 {
1344 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1345 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1346 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1347 fUpdatedMsrValue = true;
1348 }
1349
1350 if (pfAddedAndUpdated)
1351 *pfAddedAndUpdated = fUpdatedMsrValue;
1352 return VINF_SUCCESS;
1353}
1354
1355
1356/**
1357 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1358 * auto-load/store MSR area in the VMCS.
1359 *
1360 * @returns VBox status code.
1361 * @param pVCpu The cross context virtual CPU structure.
1362 * @param uMsr The MSR.
1363 */
1364static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1365{
1366 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1367 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1368 for (uint32_t i = 0; i < cMsrs; i++)
1369 {
1370 /* Find the MSR. */
1371 if (pGuestMsr->u32Msr == uMsr)
1372 {
1373 /* If it's the last MSR, simply reduce the count. */
1374 if (i == cMsrs - 1)
1375 {
1376 --cMsrs;
1377 break;
1378 }
1379
1380 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1381 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1382 pLastGuestMsr += cMsrs - 1;
1383 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1384 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1385
1386 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1387 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1388 pLastHostMsr += cMsrs - 1;
1389 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1390 pHostMsr->u64Value = pLastHostMsr->u64Value;
1391 --cMsrs;
1392 break;
1393 }
1394 pGuestMsr++;
1395 }
1396
1397 /* Update the VMCS if the count changed (meaning the MSR was found). */
1398 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1399 {
1400 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1401 AssertRCReturn(rc, rc);
1402
1403 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1404 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1405 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1406
1407 Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1408 return VINF_SUCCESS;
1409 }
1410
1411 return VERR_NOT_FOUND;
1412}
1413
1414
1415/**
1416 * Checks if the specified guest MSR is part of the auto-load/store area in
1417 * the VMCS.
1418 *
1419 * @returns true if found, false otherwise.
1420 * @param pVCpu The cross context virtual CPU structure.
1421 * @param uMsr The MSR to find.
1422 */
1423static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1424{
1425 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1426 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1427
1428 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1429 {
1430 if (pGuestMsr->u32Msr == uMsr)
1431 return true;
1432 }
1433 return false;
1434}
1435
1436
1437/**
1438 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1439 *
1440 * @param pVCpu The cross context virtual CPU structure.
1441 *
1442 * @remarks No-long-jump zone!!!
1443 */
1444static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1445{
1446 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1447 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1448 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1449 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1450
1451 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1452 {
1453 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1454
1455 /*
1456 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1457 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1458 */
1459 if (pHostMsr->u32Msr == MSR_K6_EFER)
1460 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1461 else
1462 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1463 }
1464
1465 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1466}
1467
1468
1469/**
1470 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1471 * perform lazy restoration of the host MSRs while leaving VT-x.
1472 *
1473 * @param pVCpu The cross context virtual CPU structure.
1474 *
1475 * @remarks No-long-jump zone!!!
1476 */
1477static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1478{
1479 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1480
1481 /*
1482 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1483 */
1484 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1485 {
1486 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1487#if HC_ARCH_BITS == 64
1488 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1489 {
1490 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1491 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1492 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1493 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1494 }
1495#endif
1496 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1497 }
1498}
1499
1500
1501/**
1502 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1503 * lazily while leaving VT-x.
1504 *
1505 * @returns true if it does, false otherwise.
1506 * @param pVCpu The cross context virtual CPU structure.
1507 * @param uMsr The MSR to check.
1508 */
1509static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1510{
1511 NOREF(pVCpu);
1512#if HC_ARCH_BITS == 64
1513 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1514 {
1515 switch (uMsr)
1516 {
1517 case MSR_K8_LSTAR:
1518 case MSR_K6_STAR:
1519 case MSR_K8_SF_MASK:
1520 case MSR_K8_KERNEL_GS_BASE:
1521 return true;
1522 }
1523 }
1524#else
1525 RT_NOREF(pVCpu, uMsr);
1526#endif
1527 return false;
1528}
1529
1530
1531/**
1532 * Loads a set of guests MSRs to allow read/passthru to the guest.
1533 *
1534 * The name of this function is slightly confusing. This function does NOT
1535 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1536 * common prefix for functions dealing with "lazy restoration" of the shared
1537 * MSRs.
1538 *
1539 * @param pVCpu The cross context virtual CPU structure.
1540 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1541 * out-of-sync. Make sure to update the required fields
1542 * before using them.
1543 *
1544 * @remarks No-long-jump zone!!!
1545 */
1546static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1547{
1548 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1549 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1550
1551 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1552#if HC_ARCH_BITS == 64
1553 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1554 {
1555 /*
1556 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1557 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1558 * we can skip a few MSR writes.
1559 *
1560 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1561 * guest MSR values in the guest-CPU context might be different to what's currently
1562 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1563 * CPU, see @bugref{8728}.
1564 */
1565 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1566 && pMixedCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1567 && pMixedCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1568 && pMixedCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1569 && pMixedCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1570 {
1571#ifdef VBOX_STRICT
1572 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pMixedCtx->msrKERNELGSBASE);
1573 Assert(ASMRdMsr(MSR_K8_LSTAR) == pMixedCtx->msrLSTAR);
1574 Assert(ASMRdMsr(MSR_K6_STAR) == pMixedCtx->msrSTAR);
1575 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pMixedCtx->msrSFMASK);
1576#endif
1577 }
1578 else
1579 {
1580 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1581 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1582 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1583 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1584 }
1585 }
1586#else
1587 RT_NOREF(pMixedCtx);
1588#endif
1589 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1590}
1591
1592
1593/**
1594 * Performs lazy restoration of the set of host MSRs if they were previously
1595 * loaded with guest MSR values.
1596 *
1597 * @param pVCpu The cross context virtual CPU structure.
1598 *
1599 * @remarks No-long-jump zone!!!
1600 * @remarks The guest MSRs should have been saved back into the guest-CPU
1601 * context by hmR0VmxImportGuestState()!!!
1602 */
1603static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1604{
1605 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1606 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1607
1608 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1609 {
1610 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1611#if HC_ARCH_BITS == 64
1612 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1613 {
1614 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1615 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1616 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1617 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1618 }
1619#endif
1620 }
1621 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1622}
1623
1624
1625/**
1626 * Verifies that our cached values of the VMCS controls are all
1627 * consistent with what's actually present in the VMCS.
1628 *
1629 * @returns VBox status code.
1630 * @param pVCpu The cross context virtual CPU structure.
1631 */
1632static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1633{
1634 uint32_t u32Val;
1635 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1636 AssertRCReturn(rc, rc);
1637 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1638 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1639
1640 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1641 AssertRCReturn(rc, rc);
1642 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1643 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1644
1645 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1646 AssertRCReturn(rc, rc);
1647 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1648 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1649
1650 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1651 AssertRCReturn(rc, rc);
1652 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1653 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1654
1655 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1656 {
1657 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1658 AssertRCReturn(rc, rc);
1659 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1660 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1661 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1662 }
1663
1664 return VINF_SUCCESS;
1665}
1666
1667
1668#ifdef VBOX_STRICT
1669/**
1670 * Verifies that our cached host EFER value has not changed
1671 * since we cached it.
1672 *
1673 * @param pVCpu The cross context virtual CPU structure.
1674 */
1675static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1676{
1677 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1678
1679 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1680 {
1681 uint64_t u64Val;
1682 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1683 AssertRC(rc);
1684
1685 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1686 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1687 }
1688}
1689
1690
1691/**
1692 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1693 * VMCS are correct.
1694 *
1695 * @param pVCpu The cross context virtual CPU structure.
1696 */
1697static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1698{
1699 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1700
1701 /* Verify MSR counts in the VMCS are what we think it should be. */
1702 uint32_t cMsrs;
1703 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1704 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1705
1706 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1707 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1708
1709 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1710 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1711
1712 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1713 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1714 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1715 {
1716 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1717 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1718 pGuestMsr->u32Msr, cMsrs));
1719
1720 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1721 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1722 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1723
1724 /* Verify that the permissions are as expected in the MSR bitmap. */
1725 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1726 {
1727 VMXMSREXITREAD enmRead;
1728 VMXMSREXITWRITE enmWrite;
1729 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1730 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1731 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1732 {
1733 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1734 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1735 }
1736 else
1737 {
1738 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1739 pGuestMsr->u32Msr, cMsrs));
1740 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1741 pGuestMsr->u32Msr, cMsrs));
1742 }
1743 }
1744 }
1745}
1746#endif /* VBOX_STRICT */
1747
1748
1749/**
1750 * Flushes the TLB using EPT.
1751 *
1752 * @returns VBox status code.
1753 * @param pVCpu The cross context virtual CPU structure of the calling
1754 * EMT. Can be NULL depending on @a enmFlush.
1755 * @param enmFlush Type of flush.
1756 *
1757 * @remarks Caller is responsible for making sure this function is called only
1758 * when NestedPaging is supported and providing @a enmFlush that is
1759 * supported by the CPU.
1760 * @remarks Can be called with interrupts disabled.
1761 */
1762static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1763{
1764 uint64_t au64Descriptor[2];
1765 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1766 au64Descriptor[0] = 0;
1767 else
1768 {
1769 Assert(pVCpu);
1770 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1771 }
1772 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1773
1774 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1775 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1776 rc));
1777 if ( RT_SUCCESS(rc)
1778 && pVCpu)
1779 {
1780 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1781 }
1782}
1783
1784
1785/**
1786 * Flushes the TLB using VPID.
1787 *
1788 * @returns VBox status code.
1789 * @param pVM The cross context VM structure.
1790 * @param pVCpu The cross context virtual CPU structure of the calling
1791 * EMT. Can be NULL depending on @a enmFlush.
1792 * @param enmFlush Type of flush.
1793 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1794 * on @a enmFlush).
1795 *
1796 * @remarks Can be called with interrupts disabled.
1797 */
1798static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1799{
1800 NOREF(pVM);
1801 AssertPtr(pVM);
1802 Assert(pVM->hm.s.vmx.fVpid);
1803
1804 uint64_t au64Descriptor[2];
1805 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1806 {
1807 au64Descriptor[0] = 0;
1808 au64Descriptor[1] = 0;
1809 }
1810 else
1811 {
1812 AssertPtr(pVCpu);
1813 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1814 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1815 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1816 au64Descriptor[1] = GCPtr;
1817 }
1818
1819 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1820 AssertMsg(rc == VINF_SUCCESS,
1821 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1822 if ( RT_SUCCESS(rc)
1823 && pVCpu)
1824 {
1825 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1826 }
1827}
1828
1829
1830/**
1831 * Invalidates a guest page by guest virtual address. Only relevant for
1832 * EPT/VPID, otherwise there is nothing really to invalidate.
1833 *
1834 * @returns VBox status code.
1835 * @param pVM The cross context VM structure.
1836 * @param pVCpu The cross context virtual CPU structure.
1837 * @param GCVirt Guest virtual address of the page to invalidate.
1838 */
1839VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1840{
1841 AssertPtr(pVM);
1842 AssertPtr(pVCpu);
1843 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1844
1845 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1846 if (!fFlushPending)
1847 {
1848 /*
1849 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1850 * the EPT case. See @bugref{6043} and @bugref{6177}.
1851 *
1852 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1853 * as this function maybe called in a loop with individual addresses.
1854 */
1855 if (pVM->hm.s.vmx.fVpid)
1856 {
1857 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1858
1859#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1860 /*
1861 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1862 * where executing INVVPID outside 64-bit mode does not flush translations of
1863 * 64-bit linear addresses, see @bugref{6208#c72}.
1864 */
1865 if (RT_HI_U32(GCVirt))
1866 fVpidFlush = false;
1867#endif
1868
1869 if (fVpidFlush)
1870 {
1871 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1872 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1873 }
1874 else
1875 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1876 }
1877 else if (pVM->hm.s.fNestedPaging)
1878 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1879 }
1880
1881 return VINF_SUCCESS;
1882}
1883
1884
1885/**
1886 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1887 * case where neither EPT nor VPID is supported by the CPU.
1888 *
1889 * @param pVM The cross context VM structure.
1890 * @param pVCpu The cross context virtual CPU structure.
1891 * @param pCpu Pointer to the global HM struct.
1892 *
1893 * @remarks Called with interrupts disabled.
1894 */
1895static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1896{
1897 AssertPtr(pVCpu);
1898 AssertPtr(pCpu);
1899 NOREF(pVM);
1900
1901 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1902
1903 Assert(pCpu->idCpu != NIL_RTCPUID);
1904 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1905 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1906 pVCpu->hm.s.fForceTLBFlush = false;
1907 return;
1908}
1909
1910
1911/**
1912 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1913 *
1914 * @param pVM The cross context VM structure.
1915 * @param pVCpu The cross context virtual CPU structure.
1916 * @param pCpu Pointer to the global HM CPU struct.
1917 * @remarks All references to "ASID" in this function pertains to "VPID" in
1918 * Intel's nomenclature. The reason is, to avoid confusion in compare
1919 * statements since the host-CPU copies are named "ASID".
1920 *
1921 * @remarks Called with interrupts disabled.
1922 */
1923static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1924{
1925#ifdef VBOX_WITH_STATISTICS
1926 bool fTlbFlushed = false;
1927# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1928# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1929 if (!fTlbFlushed) \
1930 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1931 } while (0)
1932#else
1933# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1934# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1935#endif
1936
1937 AssertPtr(pVM);
1938 AssertPtr(pCpu);
1939 AssertPtr(pVCpu);
1940 Assert(pCpu->idCpu != NIL_RTCPUID);
1941
1942 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1943 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1944 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1945
1946 /*
1947 * Force a TLB flush for the first world-switch if the current CPU differs from the one we
1948 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1949 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1950 * cannot reuse the current ASID anymore.
1951 */
1952 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1953 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1954 {
1955 ++pCpu->uCurrentAsid;
1956 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1957 {
1958 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1959 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1960 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1961 }
1962
1963 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1964 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1965 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1966
1967 /*
1968 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1969 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1970 */
1971 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1972 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1973 HMVMX_SET_TAGGED_TLB_FLUSHED();
1974 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1975 }
1976
1977 /* Check for explicit TLB flushes. */
1978 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1979 {
1980 /*
1981 * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
1982 * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
1983 * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
1984 * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
1985 * mappings, see @bugref{6568}.
1986 *
1987 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
1988 */
1989 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1990 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1991 HMVMX_SET_TAGGED_TLB_FLUSHED();
1992 }
1993
1994 pVCpu->hm.s.fForceTLBFlush = false;
1995 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1996
1997 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1998 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1999 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2000 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2001 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2002 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2003 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2004 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2005 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2006
2007 /* Update VMCS with the VPID. */
2008 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2009 AssertRC(rc);
2010
2011#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2012}
2013
2014
2015/**
2016 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2017 *
2018 * @returns VBox status code.
2019 * @param pVM The cross context VM structure.
2020 * @param pVCpu The cross context virtual CPU structure.
2021 * @param pCpu Pointer to the global HM CPU struct.
2022 *
2023 * @remarks Called with interrupts disabled.
2024 */
2025static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2026{
2027 AssertPtr(pVM);
2028 AssertPtr(pVCpu);
2029 AssertPtr(pCpu);
2030 Assert(pCpu->idCpu != NIL_RTCPUID);
2031 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2032 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2033
2034 /*
2035 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2036 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2037 */
2038 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2039 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2040 {
2041 pVCpu->hm.s.fForceTLBFlush = true;
2042 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2043 }
2044
2045 /* Check for explicit TLB flushes. */
2046 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2047 {
2048 pVCpu->hm.s.fForceTLBFlush = true;
2049 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2050 }
2051
2052 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2053 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2054
2055 if (pVCpu->hm.s.fForceTLBFlush)
2056 {
2057 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2058 pVCpu->hm.s.fForceTLBFlush = false;
2059 }
2060}
2061
2062
2063/**
2064 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2065 *
2066 * @returns VBox status code.
2067 * @param pVM The cross context VM structure.
2068 * @param pVCpu The cross context virtual CPU structure.
2069 * @param pCpu Pointer to the global HM CPU struct.
2070 *
2071 * @remarks Called with interrupts disabled.
2072 */
2073static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2074{
2075 AssertPtr(pVM);
2076 AssertPtr(pVCpu);
2077 AssertPtr(pCpu);
2078 Assert(pCpu->idCpu != NIL_RTCPUID);
2079 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2080 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2081
2082 /*
2083 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2084 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2085 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2086 */
2087 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2088 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2089 {
2090 pVCpu->hm.s.fForceTLBFlush = true;
2091 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2092 }
2093
2094 /* Check for explicit TLB flushes. */
2095 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2096 {
2097 /*
2098 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2099 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2100 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2101 */
2102 pVCpu->hm.s.fForceTLBFlush = true;
2103 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2104 }
2105
2106 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2107 if (pVCpu->hm.s.fForceTLBFlush)
2108 {
2109 ++pCpu->uCurrentAsid;
2110 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2111 {
2112 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2113 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2114 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2115 }
2116
2117 pVCpu->hm.s.fForceTLBFlush = false;
2118 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2119 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2120 if (pCpu->fFlushAsidBeforeUse)
2121 {
2122 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2123 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2124 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2125 {
2126 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2127 pCpu->fFlushAsidBeforeUse = false;
2128 }
2129 else
2130 {
2131 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2132 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2133 }
2134 }
2135 }
2136
2137 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2138 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2139 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2140 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2141 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2142 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2143 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2144
2145 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2146 AssertRC(rc);
2147}
2148
2149
2150/**
2151 * Flushes the guest TLB entry based on CPU capabilities.
2152 *
2153 * @param pVCpu The cross context virtual CPU structure.
2154 * @param pCpu Pointer to the global HM CPU struct.
2155 */
2156DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2157{
2158#ifdef HMVMX_ALWAYS_FLUSH_TLB
2159 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2160#endif
2161 PVM pVM = pVCpu->CTX_SUFF(pVM);
2162 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2163 {
2164 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2165 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2166 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2167 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2168 default:
2169 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2170 break;
2171 }
2172 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2173}
2174
2175
2176/**
2177 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2178 * TLB entries from the host TLB before VM-entry.
2179 *
2180 * @returns VBox status code.
2181 * @param pVM The cross context VM structure.
2182 */
2183static int hmR0VmxSetupTaggedTlb(PVM pVM)
2184{
2185 /*
2186 * Determine optimal flush type for Nested Paging.
2187 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2188 * guest execution (see hmR3InitFinalizeR0()).
2189 */
2190 if (pVM->hm.s.fNestedPaging)
2191 {
2192 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2193 {
2194 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2195 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2196 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2197 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2198 else
2199 {
2200 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2201 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2202 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2203 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2204 }
2205
2206 /* Make sure the write-back cacheable memory type for EPT is supported. */
2207 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2208 {
2209 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2210 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2211 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2212 }
2213
2214 /* EPT requires a page-walk length of 4. */
2215 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2216 {
2217 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2218 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2219 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2220 }
2221 }
2222 else
2223 {
2224 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2225 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2226 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2227 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2228 }
2229 }
2230
2231 /*
2232 * Determine optimal flush type for VPID.
2233 */
2234 if (pVM->hm.s.vmx.fVpid)
2235 {
2236 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2237 {
2238 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2239 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2240 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2241 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2242 else
2243 {
2244 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2245 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2246 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2247 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2248 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2249 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2250 pVM->hm.s.vmx.fVpid = false;
2251 }
2252 }
2253 else
2254 {
2255 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2256 Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2257 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2258 pVM->hm.s.vmx.fVpid = false;
2259 }
2260 }
2261
2262 /*
2263 * Setup the handler for flushing tagged-TLBs.
2264 */
2265 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2266 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2267 else if (pVM->hm.s.fNestedPaging)
2268 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2269 else if (pVM->hm.s.vmx.fVpid)
2270 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2271 else
2272 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2273 return VINF_SUCCESS;
2274}
2275
2276
2277/**
2278 * Sets up pin-based VM-execution controls in the VMCS.
2279 *
2280 * @returns VBox status code.
2281 * @param pVM The cross context VM structure.
2282 * @param pVCpu The cross context virtual CPU structure.
2283 */
2284static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2285{
2286 AssertPtr(pVM);
2287 AssertPtr(pVCpu);
2288
2289 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2290 uint32_t fZap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2291
2292 fVal |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2293 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2294
2295 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2296 fVal |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2297
2298 /* Enable the VMX preemption timer. */
2299 if (pVM->hm.s.vmx.fUsePreemptTimer)
2300 {
2301 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2302 fVal |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2303 }
2304
2305#if 0
2306 /* Enable posted-interrupt processing. */
2307 if (pVM->hm.s.fPostedIntrs)
2308 {
2309 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2310 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2311 fVal |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2312 }
2313#endif
2314
2315 if ((fVal & fZap) != fVal)
2316 {
2317 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2318 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, fVal, fZap));
2319 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2320 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2321 }
2322
2323 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2324 AssertRCReturn(rc, rc);
2325
2326 pVCpu->hm.s.vmx.u32PinCtls = fVal;
2327 return rc;
2328}
2329
2330
2331/**
2332 * Sets up processor-based VM-execution controls in the VMCS.
2333 *
2334 * @returns VBox status code.
2335 * @param pVM The cross context VM structure.
2336 * @param pVCpu The cross context virtual CPU structure.
2337 */
2338static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2339{
2340 AssertPtr(pVM);
2341 AssertPtr(pVCpu);
2342
2343 int rc = VERR_INTERNAL_ERROR_5;
2344 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2345 uint32_t fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2346
2347 fVal |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2348 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2349 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2350 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2351 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2352 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2353 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2354
2355 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2356 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2357 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2358 {
2359 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2360 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2361 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2362 }
2363
2364 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2365 if (!pVM->hm.s.fNestedPaging)
2366 {
2367 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2368 fVal |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2369 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2370 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2371 }
2372
2373 /* Use TPR shadowing if supported by the CPU. */
2374 if ( PDMHasApic(pVM)
2375 && pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2376 {
2377 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2378 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2379 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2380 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2381 AssertRCReturn(rc, rc);
2382
2383 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2384 /* CR8 writes cause a VM-exit based on TPR threshold. */
2385 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2386 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2387 }
2388 else
2389 {
2390 /*
2391 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2392 * Set this control only for 64-bit guests.
2393 */
2394 if (pVM->hm.s.fAllow64BitGuests)
2395 {
2396 fVal |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2397 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2398 }
2399 }
2400
2401 /* Use MSR-bitmaps if supported by the CPU. */
2402 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2403 {
2404 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2405
2406 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2407 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2408 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2409 AssertRCReturn(rc, rc);
2410
2411 /*
2412 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2413 * automatically using dedicated fields in the VMCS.
2414 */
2415 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2416 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2417 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2418 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2419 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2420
2421#if HC_ARCH_BITS == 64
2422 /*
2423 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2424 */
2425 if (pVM->hm.s.fAllow64BitGuests)
2426 {
2427 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2428 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2429 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2430 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2431 }
2432#endif
2433 /*
2434 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2435 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2436 */
2437 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2438 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2439
2440 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2441 }
2442
2443 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2444 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2445 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2446
2447 if ((fVal & fZap) != fVal)
2448 {
2449 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2450 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, fVal, fZap));
2451 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2452 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2453 }
2454
2455 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2456 AssertRCReturn(rc, rc);
2457
2458 pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2459
2460 /*
2461 * Secondary processor-based VM-execution controls.
2462 */
2463 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2464 {
2465 fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2466 fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2467
2468 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2469 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2470
2471 if (pVM->hm.s.fNestedPaging)
2472 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2473
2474 /*
2475 * Enable the INVPCID instruction if supported by the hardware and we expose
2476 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2477 */
2478 if ( (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2479 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2480 {
2481 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2482 }
2483
2484 if (pVM->hm.s.vmx.fVpid)
2485 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2486
2487 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2488 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2489
2490#if 0
2491 if (pVM->hm.s.fVirtApicRegs)
2492 {
2493 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2494 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2495
2496 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2497 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2498 }
2499#endif
2500
2501 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2502 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2503 * done dynamically. */
2504 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2505 {
2506 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2507 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2508 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2509 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2510 AssertRCReturn(rc, rc);
2511 }
2512
2513 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2514 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2515
2516 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2517 && pVM->hm.s.vmx.cPleGapTicks
2518 && pVM->hm.s.vmx.cPleWindowTicks)
2519 {
2520 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2521
2522 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2523 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2524 AssertRCReturn(rc, rc);
2525 }
2526
2527 if ((fVal & fZap) != fVal)
2528 {
2529 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2530 "cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, fVal, fZap));
2531 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2532 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2533 }
2534
2535 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2536 AssertRCReturn(rc, rc);
2537
2538 pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2539 }
2540 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2541 {
2542 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2543 "available\n"));
2544 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2545 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2546 }
2547
2548 return VINF_SUCCESS;
2549}
2550
2551
2552/**
2553 * Sets up miscellaneous (everything other than Pin & Processor-based
2554 * VM-execution) control fields in the VMCS.
2555 *
2556 * @returns VBox status code.
2557 * @param pVM The cross context VM structure.
2558 * @param pVCpu The cross context virtual CPU structure.
2559 */
2560static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2561{
2562 NOREF(pVM);
2563 AssertPtr(pVM);
2564 AssertPtr(pVCpu);
2565
2566 int rc = VERR_GENERAL_FAILURE;
2567
2568 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2569#if 0
2570 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2571 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2572 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2573
2574 /*
2575 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2576 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2577 * We thus use the exception bitmap to control it rather than use both.
2578 */
2579 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2580 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2581
2582 /* All IO & IOIO instructions cause VM-exits. */
2583 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2584 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2585
2586 /* Initialize the MSR-bitmap area. */
2587 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2588 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2589 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2590 AssertRCReturn(rc, rc);
2591#endif
2592
2593 /* Setup MSR auto-load/store area. */
2594 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2595 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2596 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2597 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2598 AssertRCReturn(rc, rc);
2599
2600 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2601 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2602 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2603 AssertRCReturn(rc, rc);
2604
2605 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2606 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2607 AssertRCReturn(rc, rc);
2608
2609 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2610#if 0
2611 /* Setup debug controls */
2612 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2613 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2614 AssertRCReturn(rc, rc);
2615#endif
2616
2617 return rc;
2618}
2619
2620
2621/**
2622 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2623 *
2624 * We shall setup those exception intercepts that don't change during the
2625 * lifetime of the VM here. The rest are done dynamically while loading the
2626 * guest state.
2627 *
2628 * @returns VBox status code.
2629 * @param pVM The cross context VM structure.
2630 * @param pVCpu The cross context virtual CPU structure.
2631 */
2632static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2633{
2634 AssertPtr(pVM);
2635 AssertPtr(pVCpu);
2636
2637 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2638
2639 uint32_t u32XcptBitmap = 0;
2640
2641 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2642 u32XcptBitmap |= RT_BIT_32(X86_XCPT_AC);
2643
2644 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2645 and writes, and because recursive #DBs can cause the CPU hang, we must always
2646 intercept #DB. */
2647 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2648
2649 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2650 if (!pVM->hm.s.fNestedPaging)
2651 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2652
2653 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2654 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2655 AssertRCReturn(rc, rc);
2656 return rc;
2657}
2658
2659
2660/**
2661 * Does per-VM VT-x initialization.
2662 *
2663 * @returns VBox status code.
2664 * @param pVM The cross context VM structure.
2665 */
2666VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2667{
2668 LogFlowFunc(("pVM=%p\n", pVM));
2669
2670 int rc = hmR0VmxStructsAlloc(pVM);
2671 if (RT_FAILURE(rc))
2672 {
2673 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2674 return rc;
2675 }
2676
2677 return VINF_SUCCESS;
2678}
2679
2680
2681/**
2682 * Does per-VM VT-x termination.
2683 *
2684 * @returns VBox status code.
2685 * @param pVM The cross context VM structure.
2686 */
2687VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2688{
2689 LogFlowFunc(("pVM=%p\n", pVM));
2690
2691#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2692 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2693 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2694#endif
2695 hmR0VmxStructsFree(pVM);
2696 return VINF_SUCCESS;
2697}
2698
2699
2700/**
2701 * Sets up the VM for execution under VT-x.
2702 * This function is only called once per-VM during initialization.
2703 *
2704 * @returns VBox status code.
2705 * @param pVM The cross context VM structure.
2706 */
2707VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2708{
2709 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2710 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2711
2712 LogFlowFunc(("pVM=%p\n", pVM));
2713
2714 /*
2715 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be
2716 * allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2717 * pRealModeTSS, see hmR3InitFinalizeR0Intel().
2718 */
2719 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2720 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2721 || !pVM->hm.s.vmx.pRealModeTSS))
2722 {
2723 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2724 return VERR_INTERNAL_ERROR;
2725 }
2726
2727 /* Initialize these always, see hmR3InitFinalizeR0().*/
2728 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2729 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2730
2731 /* Setup the tagged-TLB flush handlers. */
2732 int rc = hmR0VmxSetupTaggedTlb(pVM);
2733 if (RT_FAILURE(rc))
2734 {
2735 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2736 return rc;
2737 }
2738
2739 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2740 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2741#if HC_ARCH_BITS == 64
2742 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2743 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2744 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2745 {
2746 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2747 }
2748#endif
2749
2750 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2751 RTCCUINTREG uHostCR4 = ASMGetCR4();
2752 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2753 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2754
2755 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2756 {
2757 PVMCPU pVCpu = &pVM->aCpus[i];
2758 AssertPtr(pVCpu);
2759 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2760
2761 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2762 Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2763
2764 /* Set revision dword at the beginning of the VMCS structure. */
2765 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2766
2767 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2768 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2769 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2770 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2771
2772 /* Load this VMCS as the current VMCS. */
2773 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2774 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2775 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2776
2777 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2778 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2779 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2780
2781 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2782 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2783 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2784
2785 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2786 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2787 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2788
2789 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2790 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2791 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2792
2793#if HC_ARCH_BITS == 32
2794 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2795 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2796 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2797#endif
2798
2799 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2800 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2801 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2802 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2803
2804 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2805
2806 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2807 }
2808
2809 return VINF_SUCCESS;
2810}
2811
2812
2813/**
2814 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2815 * the VMCS.
2816 *
2817 * @returns VBox status code.
2818 */
2819static int hmR0VmxExportHostControlRegs(void)
2820{
2821 RTCCUINTREG uReg = ASMGetCR0();
2822 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2823 AssertRCReturn(rc, rc);
2824
2825 uReg = ASMGetCR3();
2826 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2827 AssertRCReturn(rc, rc);
2828
2829 uReg = ASMGetCR4();
2830 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2831 AssertRCReturn(rc, rc);
2832 return rc;
2833}
2834
2835
2836/**
2837 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2838 * the host-state area in the VMCS.
2839 *
2840 * @returns VBox status code.
2841 * @param pVCpu The cross context virtual CPU structure.
2842 */
2843static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2844{
2845#if HC_ARCH_BITS == 64
2846/**
2847 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2848 * requirements. See hmR0VmxExportHostSegmentRegs().
2849 */
2850# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2851 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2852 { \
2853 bool fValidSelector = true; \
2854 if ((selValue) & X86_SEL_LDT) \
2855 { \
2856 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2857 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2858 } \
2859 if (fValidSelector) \
2860 { \
2861 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2862 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2863 } \
2864 (selValue) = 0; \
2865 }
2866
2867 /*
2868 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2869 * should -not- save the messed up state without restoring the original host-state,
2870 * see @bugref{7240}.
2871 *
2872 * This apparently can happen (most likely the FPU changes), deal with it rather than
2873 * asserting. Was observed booting Solaris 10u10 32-bit guest.
2874 */
2875 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2876 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2877 {
2878 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2879 pVCpu->idCpu));
2880 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2881 }
2882 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2883#else
2884 RT_NOREF(pVCpu);
2885#endif
2886
2887 /*
2888 * Host DS, ES, FS and GS segment registers.
2889 */
2890#if HC_ARCH_BITS == 64
2891 RTSEL uSelDS = ASMGetDS();
2892 RTSEL uSelES = ASMGetES();
2893 RTSEL uSelFS = ASMGetFS();
2894 RTSEL uSelGS = ASMGetGS();
2895#else
2896 RTSEL uSelDS = 0;
2897 RTSEL uSelES = 0;
2898 RTSEL uSelFS = 0;
2899 RTSEL uSelGS = 0;
2900#endif
2901
2902 /*
2903 * Host CS and SS segment registers.
2904 */
2905 RTSEL uSelCS = ASMGetCS();
2906 RTSEL uSelSS = ASMGetSS();
2907
2908 /*
2909 * Host TR segment register.
2910 */
2911 RTSEL uSelTR = ASMGetTR();
2912
2913#if HC_ARCH_BITS == 64
2914 /*
2915 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2916 * gain VM-entry and restore them before we get preempted.
2917 *
2918 * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2919 */
2920 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2921 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2922 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2923 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2924# undef VMXLOCAL_ADJUST_HOST_SEG
2925#endif
2926
2927 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2928 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2929 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2930 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2931 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2932 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2933 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2934 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2935 Assert(uSelCS);
2936 Assert(uSelTR);
2937
2938 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2939#if 0
2940 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2941 Assert(uSelSS != 0);
2942#endif
2943
2944 /* Write these host selector fields into the host-state area in the VMCS. */
2945 int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2946 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2947#if HC_ARCH_BITS == 64
2948 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2949 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2950 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2951 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2952#else
2953 NOREF(uSelDS);
2954 NOREF(uSelES);
2955 NOREF(uSelFS);
2956 NOREF(uSelGS);
2957#endif
2958 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2959 AssertRCReturn(rc, rc);
2960
2961 /*
2962 * Host GDTR and IDTR.
2963 */
2964 RTGDTR Gdtr;
2965 RTIDTR Idtr;
2966 RT_ZERO(Gdtr);
2967 RT_ZERO(Idtr);
2968 ASMGetGDTR(&Gdtr);
2969 ASMGetIDTR(&Idtr);
2970 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2971 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2972 AssertRCReturn(rc, rc);
2973
2974#if HC_ARCH_BITS == 64
2975 /*
2976 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
2977 * them to the maximum limit (0xffff) on every VM-exit.
2978 */
2979 if (Gdtr.cbGdt != 0xffff)
2980 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2981
2982 /*
2983 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
2984 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the
2985 * limit as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU
2986 * behavior. However, several hosts either insists on 0xfff being the limit (Windows
2987 * Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
2988 * but botches sidt alignment in at least one consumer). So, we're only allowing the
2989 * IDTR.LIMIT to be left at 0xffff on hosts where we are sure it won't cause trouble.
2990 */
2991# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
2992 if (Idtr.cbIdt < 0x0fff)
2993# else
2994 if (Idtr.cbIdt != 0xffff)
2995# endif
2996 {
2997 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
2998 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2999 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3000 }
3001#endif
3002
3003 /*
3004 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3005 * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3006 * RPL should be too in most cases.
3007 */
3008 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3009 ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3010
3011 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3012#if HC_ARCH_BITS == 64
3013 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3014
3015 /*
3016 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3017 * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3018 * restoration if the host has something else. Task switching is not supported in 64-bit
3019 * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3020 * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3021 *
3022 * [1] See Intel spec. 3.5 "System Descriptor Types".
3023 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3024 */
3025 PVM pVM = pVCpu->CTX_SUFF(pVM);
3026 Assert(pDesc->System.u4Type == 11);
3027 if ( pDesc->System.u16LimitLow != 0x67
3028 || pDesc->System.u4LimitHigh)
3029 {
3030 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3031 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3032 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3033 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3034 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3035 }
3036
3037 /*
3038 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3039 */
3040 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3041 {
3042 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3043 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3044 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3045 {
3046 /* The GDT is read-only but the writable GDT is available. */
3047 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3048 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3049 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3050 AssertRCReturn(rc, rc);
3051 }
3052 }
3053#else
3054 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3055#endif
3056 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3057 AssertRCReturn(rc, rc);
3058
3059 /*
3060 * Host FS base and GS base.
3061 */
3062#if HC_ARCH_BITS == 64
3063 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3064 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3065 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3066 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3067 AssertRCReturn(rc, rc);
3068
3069 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3070 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3071 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3072 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3073 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3074#endif
3075 return VINF_SUCCESS;
3076}
3077
3078
3079/**
3080 * Exports certain host MSRs in the VM-exit MSR-load area and some in the
3081 * host-state area of the VMCS.
3082 *
3083 * Theses MSRs will be automatically restored on the host after every successful
3084 * VM-exit.
3085 *
3086 * @returns VBox status code.
3087 * @param pVCpu The cross context virtual CPU structure.
3088 *
3089 * @remarks No-long-jump zone!!!
3090 */
3091static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3092{
3093 AssertPtr(pVCpu);
3094 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3095
3096 /*
3097 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3098 * rather than swapping them on every VM-entry.
3099 */
3100 hmR0VmxLazySaveHostMsrs(pVCpu);
3101
3102 /*
3103 * Host Sysenter MSRs.
3104 */
3105 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3106#if HC_ARCH_BITS == 32
3107 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3108 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3109#else
3110 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3111 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3112#endif
3113 AssertRCReturn(rc, rc);
3114
3115 /*
3116 * Host EFER MSR.
3117 *
3118 * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3119 * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3120 */
3121 PVM pVM = pVCpu->CTX_SUFF(pVM);
3122 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3123 {
3124 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3125 AssertRCReturn(rc, rc);
3126 }
3127
3128 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3129
3130 return VINF_SUCCESS;
3131}
3132
3133
3134/**
3135 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3136 *
3137 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3138 * these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3139 * hmR0VMxExportGuestEntryCtls().
3140 *
3141 * @returns true if we need to load guest EFER, false otherwise.
3142 * @param pVCpu The cross context virtual CPU structure.
3143 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3144 * out-of-sync. Make sure to update the required fields
3145 * before using them.
3146 *
3147 * @remarks Requires EFER, CR4.
3148 * @remarks No-long-jump zone!!!
3149 */
3150static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3151{
3152#ifdef HMVMX_ALWAYS_SWAP_EFER
3153 return true;
3154#endif
3155
3156#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3157 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3158 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3159 return false;
3160#endif
3161
3162 PVM pVM = pVCpu->CTX_SUFF(pVM);
3163 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3164 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3165
3166 /*
3167 * For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3168 * guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3169 */
3170 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3171 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3172 {
3173 return true;
3174 }
3175
3176 /*
3177 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3178 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3179 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3180 */
3181 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3182 && (pMixedCtx->cr0 & X86_CR0_PG)
3183 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3184 {
3185 /* Assert that host is PAE capable. */
3186 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3187 return true;
3188 }
3189
3190 return false;
3191}
3192
3193
3194/**
3195 * Exports the guest state with appropriate VM-entry controls in the VMCS.
3196 *
3197 * These controls can affect things done on VM-exit; e.g. "load debug controls",
3198 * see Intel spec. 24.8.1 "VM-entry controls".
3199 *
3200 * @returns VBox status code.
3201 * @param pVCpu The cross context virtual CPU structure.
3202 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3203 * out-of-sync. Make sure to update the required fields
3204 * before using them.
3205 *
3206 * @remarks Requires EFER.
3207 * @remarks No-long-jump zone!!!
3208 */
3209static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3210{
3211 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3212 {
3213 PVM pVM = pVCpu->CTX_SUFF(pVM);
3214 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3215 uint32_t fZap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3216
3217 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3218 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3219
3220 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3221 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3222 {
3223 fVal |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3224 Log4Func(("VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n"));
3225 }
3226 else
3227 Assert(!(fVal & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3228
3229 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3230 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3231 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3232 {
3233 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3234 Log4Func(("VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n"));
3235 }
3236
3237 /*
3238 * The following should -not- be set (since we're not in SMM mode):
3239 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3240 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3241 */
3242
3243 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3244 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3245
3246 if ((fVal & fZap) != fVal)
3247 {
3248 Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3249 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, fVal, fZap));
3250 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3251 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3252 }
3253
3254 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3255 AssertRCReturn(rc, rc);
3256
3257 pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3258 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3259 }
3260 return VINF_SUCCESS;
3261}
3262
3263
3264/**
3265 * Exports the guest state with appropriate VM-exit controls in the VMCS.
3266 *
3267 * @returns VBox status code.
3268 * @param pVCpu The cross context virtual CPU structure.
3269 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3270 * out-of-sync. Make sure to update the required fields
3271 * before using them.
3272 *
3273 * @remarks Requires EFER.
3274 */
3275static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3276{
3277 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3278 {
3279 PVM pVM = pVCpu->CTX_SUFF(pVM);
3280 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3281 uint32_t fZap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3282
3283 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3284 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3285
3286 /*
3287 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3288 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3289 * hmR0VmxExportHostMsrs().
3290 */
3291#if HC_ARCH_BITS == 64
3292 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3293 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3294#else
3295 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3296 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3297 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3298 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3299 {
3300 /* The switcher returns to long mode, EFER is managed by the switcher. */
3301 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3302 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3303 }
3304 else
3305 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3306#endif
3307
3308 /* If the newer VMCS fields for managing EFER exists, use it. */
3309 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3310 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3311 {
3312 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3313 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3314 Log4Func(("VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR and VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n"));
3315 }
3316
3317 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3318 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3319
3320 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3321 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3322 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3323
3324 if ( pVM->hm.s.vmx.fUsePreemptTimer
3325 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3326 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3327
3328 if ((fVal & fZap) != fVal)
3329 {
3330 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3331 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, fVal, fZap));
3332 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3333 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3334 }
3335
3336 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3337 AssertRCReturn(rc, rc);
3338
3339 pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3340 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3341 }
3342 return VINF_SUCCESS;
3343}
3344
3345
3346/**
3347 * Sets the TPR threshold in the VMCS.
3348 *
3349 * @returns VBox status code.
3350 * @param pVCpu The cross context virtual CPU structure.
3351 * @param u32TprThreshold The TPR threshold (task-priority class only).
3352 */
3353DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3354{
3355 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3356 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3357 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3358}
3359
3360
3361/**
3362 * Exports the guest APIC TPR state into the VMCS.
3363 *
3364 * @returns VBox status code.
3365 * @param pVCpu The cross context virtual CPU structure.
3366 *
3367 * @remarks No-long-jump zone!!!
3368 */
3369static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3370{
3371 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3372 {
3373 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3374 && APICIsEnabled(pVCpu))
3375 {
3376 /*
3377 * Setup TPR shadowing.
3378 */
3379 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3380 {
3381 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3382
3383 bool fPendingIntr = false;
3384 uint8_t u8Tpr = 0;
3385 uint8_t u8PendingIntr = 0;
3386 int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3387 AssertRCReturn(rc, rc);
3388
3389 /*
3390 * If there are interrupts pending but masked by the TPR, instruct VT-x to
3391 * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3392 * priority of the pending interrupt so we can deliver the interrupt. If there
3393 * are no interrupts pending, set threshold to 0 to not cause any
3394 * TPR-below-threshold VM-exits.
3395 */
3396 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3397 uint32_t u32TprThreshold = 0;
3398 if (fPendingIntr)
3399 {
3400 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3401 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3402 const uint8_t u8TprPriority = u8Tpr >> 4;
3403 if (u8PendingPriority <= u8TprPriority)
3404 u32TprThreshold = u8PendingPriority;
3405 }
3406
3407 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3408 AssertRCReturn(rc, rc);
3409 }
3410 }
3411 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3412 }
3413 return VINF_SUCCESS;
3414}
3415
3416
3417/**
3418 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3419 *
3420 * @returns Guest's interruptibility-state.
3421 * @param pVCpu The cross context virtual CPU structure.
3422 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3423 * out-of-sync. Make sure to update the required fields
3424 * before using them.
3425 *
3426 * @remarks No-long-jump zone!!!
3427 */
3428static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3429{
3430 /*
3431 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3432 */
3433 uint32_t fIntrState = 0;
3434 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3435 {
3436 /* If inhibition is active, RIP & RFLAGS should've been accessed
3437 (i.e. read previously from the VMCS or from ring-3). */
3438#ifdef VBOX_STRICT
3439 uint64_t const fExtrn = ASMAtomicUoReadU64(&pMixedCtx->fExtrn);
3440 AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3441#endif
3442 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3443 {
3444 if (pMixedCtx->eflags.Bits.u1IF)
3445 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3446 else
3447 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3448 }
3449 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3450 {
3451 /*
3452 * We can clear the inhibit force flag as even if we go back to the recompiler
3453 * without executing guest code in VT-x, the flag's condition to be cleared is
3454 * met and thus the cleared state is correct.
3455 */
3456 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3457 }
3458 }
3459
3460 /*
3461 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3462 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3463 * setting this would block host-NMIs and IRET will not clear the blocking.
3464 *
3465 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3466 */
3467 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3468 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3469 {
3470 fIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3471 }
3472
3473 return fIntrState;
3474}
3475
3476
3477/**
3478 * Exports the guest's interruptibility-state into the guest-state area in the
3479 * VMCS.
3480 *
3481 * @returns VBox status code.
3482 * @param pVCpu The cross context virtual CPU structure.
3483 * @param fIntrState The interruptibility-state to set.
3484 */
3485static int hmR0VmxExportGuestIntrState(PVMCPU pVCpu, uint32_t fIntrState)
3486{
3487 NOREF(pVCpu);
3488 AssertMsg(!(fIntrState & 0xfffffff0), ("%#x\n", fIntrState)); /* Bits 31:4 MBZ. */
3489 Assert((fIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3490 return VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, fIntrState);
3491}
3492
3493
3494/**
3495 * Exports the exception intercepts required for guest execution in the VMCS.
3496 *
3497 * @returns VBox status code.
3498 * @param pVCpu The cross context virtual CPU structure.
3499 *
3500 * @remarks No-long-jump zone!!!
3501 */
3502static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3503{
3504 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3505 {
3506 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3507 if (pVCpu->hm.s.fGIMTrapXcptUD)
3508 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3509#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3510 else
3511 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3512#endif
3513
3514 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3515 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3516
3517 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3518 AssertRCReturn(rc, rc);
3519
3520 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3521 Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", pVCpu->hm.s.vmx.u32XcptBitmap));
3522 }
3523 return VINF_SUCCESS;
3524}
3525
3526
3527/**
3528 * Exports the guest's RIP into the guest-state area in the VMCS.
3529 *
3530 * @returns VBox status code.
3531 * @param pVCpu The cross context virtual CPU structure.
3532 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3533 * out-of-sync. Make sure to update the required fields
3534 * before using them.
3535 *
3536 * @remarks No-long-jump zone!!!
3537 */
3538static int hmR0VmxExportGuestRip(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3539{
3540 int rc = VINF_SUCCESS;
3541 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3542 {
3543 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3544 AssertRCReturn(rc, rc);
3545
3546 /* Update the exit history entry with the correct CS.BASE + RIP or just RIP. */
3547 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
3548 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
3549 else
3550 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->rip, false);
3551
3552 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3553 Log4Func(("RIP=%#RX64\n", pMixedCtx->rip));
3554 }
3555 return rc;
3556}
3557
3558
3559/**
3560 * Exports the guest's RSP into the guest-state area in the VMCS.
3561 *
3562 * @returns VBox status code.
3563 * @param pVCpu The cross context virtual CPU structure.
3564 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3565 * out-of-sync. Make sure to update the required fields
3566 * before using them.
3567 *
3568 * @remarks No-long-jump zone!!!
3569 */
3570static int hmR0VmxExportGuestRsp(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3571{
3572 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3573 {
3574 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3575 AssertRCReturn(rc, rc);
3576
3577 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3578 }
3579 return VINF_SUCCESS;
3580}
3581
3582
3583/**
3584 * Exports the guest's RFLAGS into the guest-state area in the VMCS.
3585 *
3586 * @returns VBox status code.
3587 * @param pVCpu The cross context virtual CPU structure.
3588 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3589 * out-of-sync. Make sure to update the required fields
3590 * before using them.
3591 *
3592 * @remarks No-long-jump zone!!!
3593 */
3594static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3595{
3596 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3597 {
3598 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3599 Let us assert it as such and use 32-bit VMWRITE. */
3600 Assert(!(pMixedCtx->rflags.u64 >> 32));
3601 X86EFLAGS fEFlags = pMixedCtx->eflags;
3602 Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3603 Assert(!(fEFlags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3604
3605 /*
3606 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3607 * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3608 * can run the real-mode guest code under Virtual 8086 mode.
3609 */
3610 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3611 {
3612 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3613 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3614 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3615 fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3616 fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3617 }
3618
3619 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3620 AssertRCReturn(rc, rc);
3621
3622 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3623 Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3624 }
3625 return VINF_SUCCESS;
3626}
3627
3628
3629/**
3630 * Exports the guest CR0 control register into the guest-state area in the VMCS.
3631 *
3632 * The guest FPU state is always pre-loaded hence we don't need to bother about
3633 * sharing FPU related CR0 bits between the guest and host.
3634 *
3635 * @returns VBox status code.
3636 * @param pVCpu The cross context virtual CPU structure.
3637 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3638 * out-of-sync. Make sure to update the required fields
3639 * before using them.
3640 *
3641 * @remarks No-long-jump zone!!!
3642 */
3643static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3644{
3645 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3646 {
3647 PVM pVM = pVCpu->CTX_SUFF(pVM);
3648 Assert(!RT_HI_U32(pMixedCtx->cr0));
3649 uint32_t const uShadowCR0 = pMixedCtx->cr0;
3650 uint32_t uGuestCR0 = pMixedCtx->cr0;
3651
3652 /*
3653 * Setup VT-x's view of the guest CR0.
3654 * Minimize VM-exits due to CR3 changes when we have NestedPaging.
3655 */
3656 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3657 if (pVM->hm.s.fNestedPaging)
3658 {
3659 if (CPUMIsGuestPagingEnabled(pVCpu))
3660 {
3661 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3662 uProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3663 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3664 }
3665 else
3666 {
3667 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3668 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3669 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3670 }
3671
3672 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3673 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3674 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3675 }
3676 else
3677 {
3678 /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3679 uGuestCR0 |= X86_CR0_WP;
3680 }
3681
3682 /*
3683 * Guest FPU bits.
3684 *
3685 * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3686 * using CR0.TS.
3687 *
3688 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3689 * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3690 */
3691 uGuestCR0 |= X86_CR0_NE;
3692
3693 /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3694 bool const fInterceptMF = !(uShadowCR0 & X86_CR0_NE);
3695
3696 /*
3697 * Update exception intercepts.
3698 */
3699 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3700 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3701 {
3702 Assert(PDMVmmDevHeapIsEnabled(pVM));
3703 Assert(pVM->hm.s.vmx.pRealModeTSS);
3704 uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3705 }
3706 else
3707 {
3708 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3709 uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3710 if (fInterceptMF)
3711 uXcptBitmap |= RT_BIT(X86_XCPT_MF);
3712 }
3713
3714 /* Additional intercepts for debugging, define these yourself explicitly. */
3715#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3716 uXcptBitmap |= 0
3717 | RT_BIT(X86_XCPT_BP)
3718 | RT_BIT(X86_XCPT_DE)
3719 | RT_BIT(X86_XCPT_NM)
3720 | RT_BIT(X86_XCPT_TS)
3721 | RT_BIT(X86_XCPT_UD)
3722 | RT_BIT(X86_XCPT_NP)
3723 | RT_BIT(X86_XCPT_SS)
3724 | RT_BIT(X86_XCPT_GP)
3725 | RT_BIT(X86_XCPT_PF)
3726 | RT_BIT(X86_XCPT_MF)
3727 ;
3728#elif defined(HMVMX_ALWAYS_TRAP_PF)
3729 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
3730#endif
3731 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3732 {
3733 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3734 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3735 }
3736 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3737
3738 /*
3739 * Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3740 */
3741 uint32_t fSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3742 uint32_t fZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3743 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3744 fSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3745 else
3746 Assert((fSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3747
3748 uGuestCR0 |= fSetCR0;
3749 uGuestCR0 &= fZapCR0;
3750 uGuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3751
3752 /*
3753 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3754 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3755 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3756 */
3757 uint32_t uCR0Mask = X86_CR0_PE
3758 | X86_CR0_NE
3759 | (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3760 | X86_CR0_PG
3761 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3762 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3763 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3764
3765 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3766 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3767 * and @bugref{6944}. */
3768#if 0
3769 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3770 uCr0Mask &= ~X86_CR0_PE;
3771#endif
3772 /* Update the HMCPU's copy of the CR0 mask. */
3773 pVCpu->hm.s.vmx.u32CR0Mask = uCR0Mask;
3774
3775 /*
3776 * Finally, update VMCS fields with the CR0 values.
3777 */
3778 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, uGuestCR0);
3779 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, uShadowCR0);
3780 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, uCR0Mask);
3781 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3782 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3783 AssertRCReturn(rc, rc);
3784 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3785
3786 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3787
3788 Log4Func(("uCr0Mask=%#RX32 uShadowCR0=%#RX32 uGuestCR0=%#RX32 (fSetCR0=%#RX32 fZapCR0=%#RX32\n", uCR0Mask, uShadowCR0,
3789 uGuestCR0, fSetCR0, fZapCR0));
3790 }
3791
3792 return VINF_SUCCESS;
3793}
3794
3795
3796/**
3797 * Exports the guest control registers (CR3, CR4) into the guest-state area
3798 * in the VMCS.
3799 *
3800 * @returns VBox strict status code.
3801 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3802 * without unrestricted guest access and the VMMDev is not presently
3803 * mapped (e.g. EFI32).
3804 *
3805 * @param pVCpu The cross context virtual CPU structure.
3806 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3807 * out-of-sync. Make sure to update the required fields
3808 * before using them.
3809 *
3810 * @remarks No-long-jump zone!!!
3811 */
3812static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3813{
3814 int rc = VINF_SUCCESS;
3815 PVM pVM = pVCpu->CTX_SUFF(pVM);
3816
3817 /*
3818 * Guest CR2.
3819 * It's always loaded in the assembler code. Nothing to do here.
3820 */
3821
3822 /*
3823 * Guest CR3.
3824 */
3825 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3826 {
3827 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3828 if (pVM->hm.s.fNestedPaging)
3829 {
3830 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3831
3832 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3833 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3834 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3835 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3836
3837 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3838 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3839 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3840
3841 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3842 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3843 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3844 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3845 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3846 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3847 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3848
3849 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3850 AssertRCReturn(rc, rc);
3851
3852 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3853 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3854 {
3855 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3856 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3857 {
3858 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3859 AssertRCReturn(rc, rc);
3860 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3861 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3862 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3863 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3864 AssertRCReturn(rc, rc);
3865 }
3866
3867 /*
3868 * The guest's view of its CR3 is unblemished with Nested Paging when the
3869 * guest is using paging or we have unrestricted guest execution to handle
3870 * the guest when it's not using paging.
3871 */
3872 GCPhysGuestCR3 = pMixedCtx->cr3;
3873 }
3874 else
3875 {
3876 /*
3877 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
3878 * thinks it accesses physical memory directly, we use our identity-mapped
3879 * page table to map guest-linear to guest-physical addresses. EPT takes care
3880 * of translating it to host-physical addresses.
3881 */
3882 RTGCPHYS GCPhys;
3883 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3884
3885 /* We obtain it here every time as the guest could have relocated this PCI region. */
3886 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3887 if (RT_SUCCESS(rc))
3888 { /* likely */ }
3889 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3890 {
3891 Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3892 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3893 }
3894 else
3895 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3896
3897 GCPhysGuestCR3 = GCPhys;
3898 }
3899
3900 Log4Func(("uGuestCR3=%#RGp (GstN)\n", GCPhysGuestCR3));
3901 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3902 AssertRCReturn(rc, rc);
3903 }
3904 else
3905 {
3906 /* Non-nested paging case, just use the hypervisor's CR3. */
3907 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3908
3909 Log4Func(("uGuestCR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3910 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3911 AssertRCReturn(rc, rc);
3912 }
3913
3914 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3915 }
3916
3917 /*
3918 * Guest CR4.
3919 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3920 */
3921 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3922 {
3923 Assert(!RT_HI_U32(pMixedCtx->cr4));
3924 uint32_t uGuestCR4 = pMixedCtx->cr4;
3925
3926 /* The guest's view of its CR4 is unblemished. */
3927 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, uGuestCR4);
3928 AssertRCReturn(rc, rc);
3929 Log4Func(("uShadowCR4=%#RX32\n", uGuestCR4));
3930
3931 /*
3932 * Setup VT-x's view of the guest CR4.
3933 *
3934 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3935 * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3936 * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3937 *
3938 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3939 */
3940 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3941 {
3942 Assert(pVM->hm.s.vmx.pRealModeTSS);
3943 Assert(PDMVmmDevHeapIsEnabled(pVM));
3944 uGuestCR4 &= ~X86_CR4_VME;
3945 }
3946
3947 if (pVM->hm.s.fNestedPaging)
3948 {
3949 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3950 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3951 {
3952 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3953 uGuestCR4 |= X86_CR4_PSE;
3954 /* Our identity mapping is a 32-bit page directory. */
3955 uGuestCR4 &= ~X86_CR4_PAE;
3956 }
3957 /* else use guest CR4.*/
3958 }
3959 else
3960 {
3961 /*
3962 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3963 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3964 */
3965 switch (pVCpu->hm.s.enmShadowMode)
3966 {
3967 case PGMMODE_REAL: /* Real-mode. */
3968 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3969 case PGMMODE_32_BIT: /* 32-bit paging. */
3970 {
3971 uGuestCR4 &= ~X86_CR4_PAE;
3972 break;
3973 }
3974
3975 case PGMMODE_PAE: /* PAE paging. */
3976 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3977 {
3978 uGuestCR4 |= X86_CR4_PAE;
3979 break;
3980 }
3981
3982 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3983 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3984#ifdef VBOX_ENABLE_64_BITS_GUESTS
3985 break;
3986#endif
3987 default:
3988 AssertFailed();
3989 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3990 }
3991 }
3992
3993 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3994 uint64_t fSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3995 uint64_t fZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3996 uGuestCR4 |= fSetCR4;
3997 uGuestCR4 &= fZapCR4;
3998
3999 /* Write VT-x's view of the guest CR4 into the VMCS. */
4000 Log4Func(("uGuestCR4=%#RX32 (fSetCR4=%#RX32 fZapCR4=%#RX32)\n", uGuestCR4, fSetCR4, fZapCR4));
4001 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, uGuestCR4);
4002 AssertRCReturn(rc, rc);
4003
4004 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4005 uint32_t u32CR4Mask = X86_CR4_VME
4006 | X86_CR4_PAE
4007 | X86_CR4_PGE
4008 | X86_CR4_PSE
4009 | X86_CR4_VMXE;
4010 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4011 u32CR4Mask |= X86_CR4_OSXSAVE;
4012 if (pVM->cpum.ro.GuestFeatures.fPcid)
4013 u32CR4Mask |= X86_CR4_PCIDE;
4014 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4015 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4016 AssertRCReturn(rc, rc);
4017
4018 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4019 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4020
4021 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4022 }
4023 return rc;
4024}
4025
4026
4027/**
4028 * Exports the guest debug registers into the guest-state area in the VMCS.
4029 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4030 *
4031 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4032 *
4033 * @returns VBox status code.
4034 * @param pVCpu The cross context virtual CPU structure.
4035 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4036 * out-of-sync. Make sure to update the required fields
4037 * before using them.
4038 *
4039 * @remarks No-long-jump zone!!!
4040 */
4041static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4042{
4043 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4044
4045#ifdef VBOX_STRICT
4046 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4047 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4048 {
4049 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4050 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4051 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4052 }
4053#endif
4054
4055 bool fSteppingDB = false;
4056 bool fInterceptMovDRx = false;
4057 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4058 if (pVCpu->hm.s.fSingleInstruction)
4059 {
4060 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4061 PVM pVM = pVCpu->CTX_SUFF(pVM);
4062 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4063 {
4064 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4065 Assert(fSteppingDB == false);
4066 }
4067 else
4068 {
4069 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4070 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
4071 pVCpu->hm.s.fClearTrapFlag = true;
4072 fSteppingDB = true;
4073 }
4074 }
4075
4076 uint32_t uGuestDR7;
4077 if ( fSteppingDB
4078 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4079 {
4080 /*
4081 * Use the combined guest and host DRx values found in the hypervisor register set
4082 * because the debugger has breakpoints active or someone is single stepping on the
4083 * host side without a monitor trap flag.
4084 *
4085 * Note! DBGF expects a clean DR6 state before executing guest code.
4086 */
4087#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4088 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4089 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4090 {
4091 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4092 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4093 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4094 }
4095 else
4096#endif
4097 if (!CPUMIsHyperDebugStateActive(pVCpu))
4098 {
4099 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4100 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4101 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4102 }
4103
4104 /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4105 uGuestDR7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4106 pVCpu->hm.s.fUsingHyperDR7 = true;
4107 fInterceptMovDRx = true;
4108 }
4109 else
4110 {
4111 /*
4112 * If the guest has enabled debug registers, we need to load them prior to
4113 * executing guest code so they'll trigger at the right time.
4114 */
4115 if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
4116 {
4117#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4118 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4119 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4120 {
4121 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4122 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4123 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4124 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4125 }
4126 else
4127#endif
4128 if (!CPUMIsGuestDebugStateActive(pVCpu))
4129 {
4130 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4131 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4132 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4133 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4134 }
4135 Assert(!fInterceptMovDRx);
4136 }
4137 /*
4138 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4139 * must intercept #DB in order to maintain a correct DR6 guest value, and
4140 * because we need to intercept it to prevent nested #DBs from hanging the
4141 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4142 */
4143#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4144 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4145 && !CPUMIsGuestDebugStateActive(pVCpu))
4146#else
4147 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4148#endif
4149 {
4150 fInterceptMovDRx = true;
4151 }
4152
4153 /* Update DR7 with the actual guest value. */
4154 uGuestDR7 = pMixedCtx->dr[7];
4155 pVCpu->hm.s.fUsingHyperDR7 = false;
4156 }
4157
4158 if (fInterceptMovDRx)
4159 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4160 else
4161 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4162
4163 /*
4164 * Update the processor-based VM-execution controls for MOV-DRx intercepts and the monitor-trap flag.
4165 */
4166 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4167 {
4168 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4169 AssertRCReturn(rc2, rc2);
4170 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4171 }
4172
4173 /*
4174 * Update guest DR7.
4175 */
4176 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, uGuestDR7);
4177 AssertRCReturn(rc, rc);
4178
4179 return VINF_SUCCESS;
4180}
4181
4182
4183#ifdef VBOX_STRICT
4184/**
4185 * Strict function to validate segment registers.
4186 *
4187 * @remarks Will import guest CR0 on strict builds during validation of
4188 * segments.
4189 */
4190static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCCPUMCTX pCtx)
4191{
4192 /*
4193 * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4194 *
4195 * The reason we check for attribute value 0 in this function and not just the unusable bit is
4196 * because hmR0VmxWriteSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4197 * and doesn't change the guest-context value.
4198 */
4199 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4200 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4201 && ( !CPUMIsGuestInRealModeEx(pCtx)
4202 && !CPUMIsGuestInV86ModeEx(pCtx)))
4203 {
4204 /* Protected mode checks */
4205 /* CS */
4206 Assert(pCtx->cs.Attr.n.u1Present);
4207 Assert(!(pCtx->cs.Attr.u & 0xf00));
4208 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4209 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4210 || !(pCtx->cs.Attr.n.u1Granularity));
4211 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4212 || (pCtx->cs.Attr.n.u1Granularity));
4213 /* CS cannot be loaded with NULL in protected mode. */
4214 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4215 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4216 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4217 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4218 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4219 else
4220 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4221 /* SS */
4222 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4223 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4224 if ( !(pCtx->cr0 & X86_CR0_PE)
4225 || pCtx->cs.Attr.n.u4Type == 3)
4226 {
4227 Assert(!pCtx->ss.Attr.n.u2Dpl);
4228 }
4229 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4230 {
4231 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4232 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4233 Assert(pCtx->ss.Attr.n.u1Present);
4234 Assert(!(pCtx->ss.Attr.u & 0xf00));
4235 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4236 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4237 || !(pCtx->ss.Attr.n.u1Granularity));
4238 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4239 || (pCtx->ss.Attr.n.u1Granularity));
4240 }
4241 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4242 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4243 {
4244 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4245 Assert(pCtx->ds.Attr.n.u1Present);
4246 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4247 Assert(!(pCtx->ds.Attr.u & 0xf00));
4248 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4249 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4250 || !(pCtx->ds.Attr.n.u1Granularity));
4251 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4252 || (pCtx->ds.Attr.n.u1Granularity));
4253 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4254 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4255 }
4256 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4257 {
4258 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4259 Assert(pCtx->es.Attr.n.u1Present);
4260 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4261 Assert(!(pCtx->es.Attr.u & 0xf00));
4262 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4263 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4264 || !(pCtx->es.Attr.n.u1Granularity));
4265 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4266 || (pCtx->es.Attr.n.u1Granularity));
4267 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4268 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4269 }
4270 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4271 {
4272 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4273 Assert(pCtx->fs.Attr.n.u1Present);
4274 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4275 Assert(!(pCtx->fs.Attr.u & 0xf00));
4276 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4277 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4278 || !(pCtx->fs.Attr.n.u1Granularity));
4279 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4280 || (pCtx->fs.Attr.n.u1Granularity));
4281 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4282 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4283 }
4284 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4285 {
4286 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4287 Assert(pCtx->gs.Attr.n.u1Present);
4288 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4289 Assert(!(pCtx->gs.Attr.u & 0xf00));
4290 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4291 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4292 || !(pCtx->gs.Attr.n.u1Granularity));
4293 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4294 || (pCtx->gs.Attr.n.u1Granularity));
4295 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4296 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4297 }
4298 /* 64-bit capable CPUs. */
4299# if HC_ARCH_BITS == 64
4300 Assert(!(pCtx->cs.u64Base >> 32));
4301 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4302 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4303 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4304# endif
4305 }
4306 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4307 || ( CPUMIsGuestInRealModeEx(pCtx)
4308 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4309 {
4310 /* Real and v86 mode checks. */
4311 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4312 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4313 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4314 {
4315 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4316 }
4317 else
4318 {
4319 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4320 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4321 }
4322
4323 /* CS */
4324 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4325 Assert(pCtx->cs.u32Limit == 0xffff);
4326 Assert(u32CSAttr == 0xf3);
4327 /* SS */
4328 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4329 Assert(pCtx->ss.u32Limit == 0xffff);
4330 Assert(u32SSAttr == 0xf3);
4331 /* DS */
4332 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4333 Assert(pCtx->ds.u32Limit == 0xffff);
4334 Assert(u32DSAttr == 0xf3);
4335 /* ES */
4336 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4337 Assert(pCtx->es.u32Limit == 0xffff);
4338 Assert(u32ESAttr == 0xf3);
4339 /* FS */
4340 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4341 Assert(pCtx->fs.u32Limit == 0xffff);
4342 Assert(u32FSAttr == 0xf3);
4343 /* GS */
4344 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4345 Assert(pCtx->gs.u32Limit == 0xffff);
4346 Assert(u32GSAttr == 0xf3);
4347 /* 64-bit capable CPUs. */
4348# if HC_ARCH_BITS == 64
4349 Assert(!(pCtx->cs.u64Base >> 32));
4350 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4351 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4352 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4353# endif
4354 }
4355}
4356#endif /* VBOX_STRICT */
4357
4358
4359/**
4360 * Writes a guest segment register into the guest-state area in the VMCS.
4361 *
4362 * @returns VBox status code.
4363 * @param pVCpu The cross context virtual CPU structure.
4364 * @param idxSel Index of the selector in the VMCS.
4365 * @param idxLimit Index of the segment limit in the VMCS.
4366 * @param idxBase Index of the segment base in the VMCS.
4367 * @param idxAccess Index of the access rights of the segment in the VMCS.
4368 * @param pSelReg Pointer to the segment selector.
4369 *
4370 * @remarks No-long-jump zone!!!
4371 */
4372static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4373 uint32_t idxAccess, PCCPUMSELREG pSelReg)
4374{
4375 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4376 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4377 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4378 AssertRCReturn(rc, rc);
4379
4380 uint32_t u32Access = pSelReg->Attr.u;
4381 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4382 {
4383 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4384 u32Access = 0xf3;
4385 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4386 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4387 }
4388 else
4389 {
4390 /*
4391 * The way to differentiate between whether this is really a null selector or was just
4392 * a selector loaded with 0 in real-mode is using the segment attributes. A selector
4393 * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4394 * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4395 * NULL selectors loaded in protected-mode have their attribute as 0.
4396 */
4397 if (!u32Access)
4398 u32Access = X86DESCATTR_UNUSABLE;
4399 }
4400
4401 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4402 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4403 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4404
4405 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4406 AssertRCReturn(rc, rc);
4407 return rc;
4408}
4409
4410
4411/**
4412 * Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4413 * into the guest-state area in the VMCS.
4414 *
4415 * @returns VBox status code.
4416 * @param pVCpu The cross context virtual CPU structure.
4417 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4418 * out-of-sync. Make sure to update the required fields
4419 * before using them.
4420 *
4421 * @remarks Will import guest CR0 on strict builds during validation of
4422 * segments.
4423 * @remarks No-long-jump zone!!!
4424 */
4425static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4426{
4427 int rc = VERR_INTERNAL_ERROR_5;
4428 PVM pVM = pVCpu->CTX_SUFF(pVM);
4429
4430 /*
4431 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4432 */
4433 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4434 {
4435 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4436 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4437 {
4438 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4439 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4440 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4441 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4442 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4443 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4444 }
4445
4446#ifdef VBOX_WITH_REM
4447 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4448 {
4449 Assert(pVM->hm.s.vmx.pRealModeTSS);
4450 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4451 if ( pVCpu->hm.s.vmx.fWasInRealMode
4452 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4453 {
4454 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4455 in real-mode (e.g. OpenBSD 4.0) */
4456 REMFlushTBs(pVM);
4457 Log4Func(("Switch to protected mode detected!\n"));
4458 pVCpu->hm.s.vmx.fWasInRealMode = false;
4459 }
4460 }
4461#endif
4462 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4463 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4464 AssertRCReturn(rc, rc);
4465 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4466 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4467 AssertRCReturn(rc, rc);
4468 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4469 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4470 AssertRCReturn(rc, rc);
4471 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4472 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4473 AssertRCReturn(rc, rc);
4474 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4475 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4476 AssertRCReturn(rc, rc);
4477 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4478 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4479 AssertRCReturn(rc, rc);
4480
4481#ifdef VBOX_STRICT
4482 /* Validate. */
4483 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4484#endif
4485
4486 /* Update the exit history entry with the correct CS.BASE + RIP. */
4487 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4488 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
4489
4490 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SREG_MASK);
4491 Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4492 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4493 }
4494
4495 /*
4496 * Guest TR.
4497 */
4498 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4499 {
4500 /*
4501 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4502 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4503 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4504 */
4505 uint16_t u16Sel = 0;
4506 uint32_t u32Limit = 0;
4507 uint64_t u64Base = 0;
4508 uint32_t u32AccessRights = 0;
4509
4510 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4511 {
4512 u16Sel = pMixedCtx->tr.Sel;
4513 u32Limit = pMixedCtx->tr.u32Limit;
4514 u64Base = pMixedCtx->tr.u64Base;
4515 u32AccessRights = pMixedCtx->tr.Attr.u;
4516 }
4517 else
4518 {
4519 Assert(pVM->hm.s.vmx.pRealModeTSS);
4520 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4521
4522 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4523 RTGCPHYS GCPhys;
4524 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4525 AssertRCReturn(rc, rc);
4526
4527 X86DESCATTR DescAttr;
4528 DescAttr.u = 0;
4529 DescAttr.n.u1Present = 1;
4530 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4531
4532 u16Sel = 0;
4533 u32Limit = HM_VTX_TSS_SIZE;
4534 u64Base = GCPhys; /* in real-mode phys = virt. */
4535 u32AccessRights = DescAttr.u;
4536 }
4537
4538 /* Validate. */
4539 Assert(!(u16Sel & RT_BIT(2)));
4540 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4541 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4542 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4543 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4544 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4545 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4546 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4547 Assert( (u32Limit & 0xfff) == 0xfff
4548 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4549 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4550 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4551
4552 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4553 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4554 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4555 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4556 AssertRCReturn(rc, rc);
4557
4558 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4559 Log4Func(("TR base=%#RX64\n", pMixedCtx->tr.u64Base));
4560 }
4561
4562 /*
4563 * Guest GDTR.
4564 */
4565 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4566 {
4567 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4568 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4569 AssertRCReturn(rc, rc);
4570
4571 /* Validate. */
4572 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4573
4574 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4575 Log4Func(("GDTR base=%#RX64\n", pMixedCtx->gdtr.pGdt));
4576 }
4577
4578 /*
4579 * Guest LDTR.
4580 */
4581 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4582 {
4583 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4584 uint32_t u32Access = 0;
4585 if (!pMixedCtx->ldtr.Attr.u)
4586 u32Access = X86DESCATTR_UNUSABLE;
4587 else
4588 u32Access = pMixedCtx->ldtr.Attr.u;
4589
4590 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4591 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4592 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4593 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4594 AssertRCReturn(rc, rc);
4595
4596 /* Validate. */
4597 if (!(u32Access & X86DESCATTR_UNUSABLE))
4598 {
4599 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4600 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4601 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4602 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4603 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4604 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4605 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4606 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4607 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4608 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4609 }
4610
4611 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4612 Log4Func(("LDTR base=%#RX64\n", pMixedCtx->ldtr.u64Base));
4613 }
4614
4615 /*
4616 * Guest IDTR.
4617 */
4618 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4619 {
4620 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4621 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4622 AssertRCReturn(rc, rc);
4623
4624 /* Validate. */
4625 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4626
4627 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4628 Log4Func(("IDTR base=%#RX64\n", pMixedCtx->idtr.pIdt));
4629 }
4630
4631 return VINF_SUCCESS;
4632}
4633
4634
4635/**
4636 * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4637 * areas.
4638 *
4639 * These MSRs will automatically be loaded to the host CPU on every successful
4640 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4641 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4642 * -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4643 *
4644 * Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4645 *
4646 * @returns VBox status code.
4647 * @param pVCpu The cross context virtual CPU structure.
4648 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4649 * out-of-sync. Make sure to update the required fields
4650 * before using them.
4651 *
4652 * @remarks No-long-jump zone!!!
4653 */
4654static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4655{
4656 AssertPtr(pVCpu);
4657 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4658
4659 /*
4660 * MSRs that we use the auto-load/store MSR area in the VMCS.
4661 * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4662 */
4663 PVM pVM = pVCpu->CTX_SUFF(pVM);
4664 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4665 {
4666 if (pVM->hm.s.fAllow64BitGuests)
4667 {
4668#if HC_ARCH_BITS == 32
4669 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4670 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4671 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4672 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4673 AssertRCReturn(rc, rc);
4674# ifdef LOG_ENABLED
4675 PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)CpVCpu->hm.s.vmx.pvGuestMsr;
4676 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4677 Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4678# endif
4679#endif
4680 }
4681 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4682 }
4683
4684 /*
4685 * Guest Sysenter MSRs.
4686 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4687 * VM-exits on WRMSRs for these MSRs.
4688 */
4689 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4690 {
4691 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4692 {
4693 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs);
4694 AssertRCReturn(rc, rc);
4695 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4696 }
4697
4698 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4699 {
4700 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip);
4701 AssertRCReturn(rc, rc);
4702 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4703 }
4704
4705 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4706 {
4707 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp);
4708 AssertRCReturn(rc, rc);
4709 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4710 }
4711 }
4712
4713 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4714 {
4715 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4716 {
4717 /*
4718 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4719 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4720 */
4721 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4722 {
4723 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4724 AssertRCReturn(rc,rc);
4725 Log4Func(("EFER=%#RX64\n", pMixedCtx->msrEFER));
4726 }
4727 else
4728 {
4729 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4730 NULL /* pfAddedAndUpdated */);
4731 AssertRCReturn(rc, rc);
4732
4733 /* We need to intercept reads too, see @bugref{7386#c16}. */
4734 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4735 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4736 Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pMixedCtx->msrEFER,
4737 pVCpu->hm.s.vmx.cMsrs));
4738 }
4739 }
4740 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4741 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4742 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4743 }
4744
4745 return VINF_SUCCESS;
4746}
4747
4748
4749#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4750/**
4751 * Check if guest state allows safe use of 32-bit switcher again.
4752 *
4753 * Segment bases and protected mode structures must be 32-bit addressable
4754 * because the 32-bit switcher will ignore high dword when writing these VMCS
4755 * fields. See @bugref{8432} for details.
4756 *
4757 * @returns true if safe, false if must continue to use the 64-bit switcher.
4758 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4759 * out-of-sync. Make sure to update the required fields
4760 * before using them.
4761 *
4762 * @remarks No-long-jump zone!!!
4763 */
4764static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pMixedCtx)
4765{
4766 if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4767 if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4768 if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4769 if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4770 if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4771 if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4772 if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4773 if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4774 if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4775 if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4776
4777 /* All good, bases are 32-bit. */
4778 return true;
4779}
4780#endif
4781
4782
4783/**
4784 * Selects up the appropriate function to run guest code.
4785 *
4786 * @returns VBox status code.
4787 * @param pVCpu The cross context virtual CPU structure.
4788 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4789 * out-of-sync. Make sure to update the required fields
4790 * before using them.
4791 *
4792 * @remarks No-long-jump zone!!!
4793 */
4794static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4795{
4796 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4797 {
4798#ifndef VBOX_ENABLE_64_BITS_GUESTS
4799 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4800#endif
4801 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4802#if HC_ARCH_BITS == 32
4803 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4804 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4805 {
4806#ifdef VBOX_STRICT
4807 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4808 {
4809 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4810 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4811 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4812 | HM_CHANGED_VMX_ENTRY_CTLS
4813 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4814 }
4815#endif
4816 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4817
4818 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4819 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4820 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4821 Log4Func(("Selected 64-bit switcher\n"));
4822 }
4823#else
4824 /* 64-bit host. */
4825 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4826#endif
4827 }
4828 else
4829 {
4830 /* Guest is not in long mode, use the 32-bit handler. */
4831#if HC_ARCH_BITS == 32
4832 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4833 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4834 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4835 {
4836# ifdef VBOX_STRICT
4837 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4838 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4839 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4840 | HM_CHANGED_VMX_ENTRY_CTLS
4841 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4842# endif
4843 }
4844# ifdef VBOX_ENABLE_64_BITS_GUESTS
4845 /*
4846 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4847 * design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4848 * switcher flag because now we know the guest is in a sane state where it's safe
4849 * to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4850 * the much faster 32-bit switcher again.
4851 */
4852 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4853 {
4854 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4855 Log4Func(("Selected 32-bit switcher\n"));
4856 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4857 }
4858 else
4859 {
4860 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4861 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4862 || hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
4863 {
4864 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4865 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4866 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4867 | HM_CHANGED_VMX_ENTRY_CTLS
4868 | HM_CHANGED_VMX_EXIT_CTLS
4869 | HM_CHANGED_HOST_CONTEXT);
4870 Log4Func(("Selected 32-bit switcher (safe)\n"));
4871 }
4872 }
4873# else
4874 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4875# endif
4876#else
4877 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4878#endif
4879 }
4880 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4881 return VINF_SUCCESS;
4882}
4883
4884
4885/**
4886 * Wrapper for running the guest code in VT-x.
4887 *
4888 * @returns VBox status code, no informational status codes.
4889 * @param pVM The cross context VM structure.
4890 * @param pVCpu The cross context virtual CPU structure.
4891 * @param pCtx Pointer to the guest-CPU context.
4892 *
4893 * @remarks No-long-jump zone!!!
4894 */
4895DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4896{
4897 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4898 pCtx->fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
4899
4900 /*
4901 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4902 * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4903 * callee-saved and thus the need for this XMM wrapper.
4904 *
4905 * See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4906 */
4907 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4908 /** @todo Add stats for resume vs launch. */
4909#ifdef VBOX_WITH_KERNEL_USING_XMM
4910 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4911#else
4912 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4913#endif
4914 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4915 return rc;
4916}
4917
4918
4919/**
4920 * Reports world-switch error and dumps some useful debug info.
4921 *
4922 * @param pVM The cross context VM structure.
4923 * @param pVCpu The cross context virtual CPU structure.
4924 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4925 * @param pCtx Pointer to the guest-CPU context.
4926 * @param pVmxTransient Pointer to the VMX transient structure (only
4927 * exitReason updated).
4928 */
4929static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4930{
4931 Assert(pVM);
4932 Assert(pVCpu);
4933 Assert(pCtx);
4934 Assert(pVmxTransient);
4935 HMVMX_ASSERT_PREEMPT_SAFE();
4936
4937 Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
4938 switch (rcVMRun)
4939 {
4940 case VERR_VMX_INVALID_VMXON_PTR:
4941 AssertFailed();
4942 break;
4943 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4944 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4945 {
4946 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4947 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4948 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4949 AssertRC(rc);
4950
4951 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4952 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4953 Cannot do it here as we may have been long preempted. */
4954
4955#ifdef VBOX_STRICT
4956 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4957 pVmxTransient->uExitReason));
4958 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4959 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4960 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4961 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4962 else
4963 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4964 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4965 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4966
4967 /* VMX control bits. */
4968 uint32_t u32Val;
4969 uint64_t u64Val;
4970 RTHCUINTREG uHCReg;
4971 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4972 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4973 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4974 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4975 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
4976 {
4977 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4978 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4979 }
4980 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4981 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4982 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4983 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4984 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4985 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4986 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4987 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4988 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4989 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4990 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4991 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4992 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4993 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4994 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4995 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4996 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4997 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4998 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4999 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5000 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5001 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5002 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5003 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5004 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5005 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5006 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5007 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5008 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5009 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5010 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5011 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5012 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5013 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5014 if (pVM->hm.s.fNestedPaging)
5015 {
5016 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5017 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5018 }
5019
5020 /* Guest bits. */
5021 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5022 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5023 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5024 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5025 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5026 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5027 if (pVM->hm.s.vmx.fVpid)
5028 {
5029 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5030 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5031 }
5032
5033 /* Host bits. */
5034 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5035 Log4(("Host CR0 %#RHr\n", uHCReg));
5036 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5037 Log4(("Host CR3 %#RHr\n", uHCReg));
5038 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5039 Log4(("Host CR4 %#RHr\n", uHCReg));
5040
5041 RTGDTR HostGdtr;
5042 PCX86DESCHC pDesc;
5043 ASMGetGDTR(&HostGdtr);
5044 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5045 Log4(("Host CS %#08x\n", u32Val));
5046 if (u32Val < HostGdtr.cbGdt)
5047 {
5048 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5049 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5050 }
5051
5052 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5053 Log4(("Host DS %#08x\n", u32Val));
5054 if (u32Val < HostGdtr.cbGdt)
5055 {
5056 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5057 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5058 }
5059
5060 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5061 Log4(("Host ES %#08x\n", u32Val));
5062 if (u32Val < HostGdtr.cbGdt)
5063 {
5064 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5065 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5066 }
5067
5068 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5069 Log4(("Host FS %#08x\n", u32Val));
5070 if (u32Val < HostGdtr.cbGdt)
5071 {
5072 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5073 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5074 }
5075
5076 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5077 Log4(("Host GS %#08x\n", u32Val));
5078 if (u32Val < HostGdtr.cbGdt)
5079 {
5080 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5081 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5082 }
5083
5084 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5085 Log4(("Host SS %#08x\n", u32Val));
5086 if (u32Val < HostGdtr.cbGdt)
5087 {
5088 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5089 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5090 }
5091
5092 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5093 Log4(("Host TR %#08x\n", u32Val));
5094 if (u32Val < HostGdtr.cbGdt)
5095 {
5096 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5097 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5098 }
5099
5100 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5101 Log4(("Host TR Base %#RHv\n", uHCReg));
5102 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5103 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5104 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5105 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5106 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5107 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5108 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5109 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5110 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5111 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5112 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5113 Log4(("Host RSP %#RHv\n", uHCReg));
5114 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5115 Log4(("Host RIP %#RHv\n", uHCReg));
5116# if HC_ARCH_BITS == 64
5117 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5118 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5119 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5120 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5121 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5122 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5123# endif
5124#endif /* VBOX_STRICT */
5125 break;
5126 }
5127
5128 default:
5129 /* Impossible */
5130 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5131 break;
5132 }
5133 NOREF(pVM); NOREF(pCtx);
5134}
5135
5136
5137#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5138#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5139# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5140#endif
5141#ifdef VBOX_STRICT
5142static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5143{
5144 switch (idxField)
5145 {
5146 case VMX_VMCS_GUEST_RIP:
5147 case VMX_VMCS_GUEST_RSP:
5148 case VMX_VMCS_GUEST_SYSENTER_EIP:
5149 case VMX_VMCS_GUEST_SYSENTER_ESP:
5150 case VMX_VMCS_GUEST_GDTR_BASE:
5151 case VMX_VMCS_GUEST_IDTR_BASE:
5152 case VMX_VMCS_GUEST_CS_BASE:
5153 case VMX_VMCS_GUEST_DS_BASE:
5154 case VMX_VMCS_GUEST_ES_BASE:
5155 case VMX_VMCS_GUEST_FS_BASE:
5156 case VMX_VMCS_GUEST_GS_BASE:
5157 case VMX_VMCS_GUEST_SS_BASE:
5158 case VMX_VMCS_GUEST_LDTR_BASE:
5159 case VMX_VMCS_GUEST_TR_BASE:
5160 case VMX_VMCS_GUEST_CR3:
5161 return true;
5162 }
5163 return false;
5164}
5165
5166static bool hmR0VmxIsValidReadField(uint32_t idxField)
5167{
5168 switch (idxField)
5169 {
5170 /* Read-only fields. */
5171 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5172 return true;
5173 }
5174 /* Remaining readable fields should also be writable. */
5175 return hmR0VmxIsValidWriteField(idxField);
5176}
5177#endif /* VBOX_STRICT */
5178
5179
5180/**
5181 * Executes the specified handler in 64-bit mode.
5182 *
5183 * @returns VBox status code (no informational status codes).
5184 * @param pVCpu The cross context virtual CPU structure.
5185 * @param enmOp The operation to perform.
5186 * @param cParams Number of parameters.
5187 * @param paParam Array of 32-bit parameters.
5188 */
5189VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp,
5190 uint32_t cParams, uint32_t *paParam)
5191{
5192 PVM pVM = pVCpu->CTX_SUFF(pVM);
5193 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5194 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5195 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5196 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5197
5198#ifdef VBOX_STRICT
5199 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5200 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5201
5202 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5203 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5204#endif
5205
5206 /* Disable interrupts. */
5207 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5208
5209#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5210 RTCPUID idHostCpu = RTMpCpuId();
5211 CPUMR0SetLApic(pVCpu, idHostCpu);
5212#endif
5213
5214 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5215 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5216
5217 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5218 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5219 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5220
5221 /* Leave VMX Root Mode. */
5222 VMXDisable();
5223
5224 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5225
5226 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5227 CPUMSetHyperEIP(pVCpu, enmOp);
5228 for (int i = (int)cParams - 1; i >= 0; i--)
5229 CPUMPushHyper(pVCpu, paParam[i]);
5230
5231 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5232
5233 /* Call the switcher. */
5234 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5235 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5236
5237 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5238 /* Make sure the VMX instructions don't cause #UD faults. */
5239 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5240
5241 /* Re-enter VMX Root Mode */
5242 int rc2 = VMXEnable(HCPhysCpuPage);
5243 if (RT_FAILURE(rc2))
5244 {
5245 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5246 ASMSetFlags(fOldEFlags);
5247 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5248 return rc2;
5249 }
5250
5251 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5252 AssertRC(rc2);
5253 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5254 Assert(!(ASMGetFlags() & X86_EFL_IF));
5255 ASMSetFlags(fOldEFlags);
5256 return rc;
5257}
5258
5259
5260/**
5261 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5262 * supporting 64-bit guests.
5263 *
5264 * @returns VBox status code.
5265 * @param fResume Whether to VMLAUNCH or VMRESUME.
5266 * @param pCtx Pointer to the guest-CPU context.
5267 * @param pCache Pointer to the VMCS cache.
5268 * @param pVM The cross context VM structure.
5269 * @param pVCpu The cross context virtual CPU structure.
5270 */
5271DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5272{
5273 NOREF(fResume);
5274
5275 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5276 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5277
5278#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5279 pCache->uPos = 1;
5280 pCache->interPD = PGMGetInterPaeCR3(pVM);
5281 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5282#endif
5283
5284#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5285 pCache->TestIn.HCPhysCpuPage = 0;
5286 pCache->TestIn.HCPhysVmcs = 0;
5287 pCache->TestIn.pCache = 0;
5288 pCache->TestOut.HCPhysVmcs = 0;
5289 pCache->TestOut.pCache = 0;
5290 pCache->TestOut.pCtx = 0;
5291 pCache->TestOut.eflags = 0;
5292#else
5293 NOREF(pCache);
5294#endif
5295
5296 uint32_t aParam[10];
5297 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5298 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5299 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5300 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5301 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5302 aParam[5] = 0;
5303 aParam[6] = VM_RC_ADDR(pVM, pVM);
5304 aParam[7] = 0;
5305 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5306 aParam[9] = 0;
5307
5308#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5309 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5310 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5311#endif
5312 int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5313
5314#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5315 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5316 Assert(pCtx->dr[4] == 10);
5317 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5318#endif
5319
5320#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5321 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5322 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5323 pVCpu->hm.s.vmx.HCPhysVmcs));
5324 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5325 pCache->TestOut.HCPhysVmcs));
5326 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5327 pCache->TestOut.pCache));
5328 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5329 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5330 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5331 pCache->TestOut.pCtx));
5332 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5333#endif
5334 return rc;
5335}
5336
5337
5338/**
5339 * Initialize the VMCS-Read cache.
5340 *
5341 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5342 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5343 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5344 * (those that have a 32-bit FULL & HIGH part).
5345 *
5346 * @returns VBox status code.
5347 * @param pVM The cross context VM structure.
5348 * @param pVCpu The cross context virtual CPU structure.
5349 */
5350static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5351{
5352#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5353{ \
5354 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5355 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5356 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5357 ++cReadFields; \
5358}
5359
5360 AssertPtr(pVM);
5361 AssertPtr(pVCpu);
5362 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5363 uint32_t cReadFields = 0;
5364
5365 /*
5366 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5367 * and serve to indicate exceptions to the rules.
5368 */
5369
5370 /* Guest-natural selector base fields. */
5371#if 0
5372 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5373 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5374 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5375#endif
5376 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5377 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5378 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5379 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5380 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5381 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5382 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5383 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5384 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5385 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5386 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5387 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5388#if 0
5389 /* Unused natural width guest-state fields. */
5390 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5391 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5392#endif
5393 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5394 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5395
5396 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5397 these 64-bit fields (using "FULL" and "HIGH" fields). */
5398#if 0
5399 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5400 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5401 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5402 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5403 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5404 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5405 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5406 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5407 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5408#endif
5409
5410 /* Natural width guest-state fields. */
5411 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5412#if 0
5413 /* Currently unused field. */
5414 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5415#endif
5416
5417 if (pVM->hm.s.fNestedPaging)
5418 {
5419 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5420 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5421 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5422 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5423 }
5424 else
5425 {
5426 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5427 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5428 }
5429
5430#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5431 return VINF_SUCCESS;
5432}
5433
5434
5435/**
5436 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5437 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5438 * darwin, running 64-bit guests).
5439 *
5440 * @returns VBox status code.
5441 * @param pVCpu The cross context virtual CPU structure.
5442 * @param idxField The VMCS field encoding.
5443 * @param u64Val 16, 32 or 64-bit value.
5444 */
5445VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5446{
5447 int rc;
5448 switch (idxField)
5449 {
5450 /*
5451 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5452 */
5453 /* 64-bit Control fields. */
5454 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5455 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5456 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5457 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5458 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5459 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5460 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5461 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5462 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5463 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5464 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5465 case VMX_VMCS64_CTRL_EPTP_FULL:
5466 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5467 /* 64-bit Guest-state fields. */
5468 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5469 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5470 case VMX_VMCS64_GUEST_PAT_FULL:
5471 case VMX_VMCS64_GUEST_EFER_FULL:
5472 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5473 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5474 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5475 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5476 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5477 /* 64-bit Host-state fields. */
5478 case VMX_VMCS64_HOST_PAT_FULL:
5479 case VMX_VMCS64_HOST_EFER_FULL:
5480 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5481 {
5482 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5483 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5484 break;
5485 }
5486
5487 /*
5488 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5489 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5490 */
5491 /* Natural-width Guest-state fields. */
5492 case VMX_VMCS_GUEST_CR3:
5493 case VMX_VMCS_GUEST_ES_BASE:
5494 case VMX_VMCS_GUEST_CS_BASE:
5495 case VMX_VMCS_GUEST_SS_BASE:
5496 case VMX_VMCS_GUEST_DS_BASE:
5497 case VMX_VMCS_GUEST_FS_BASE:
5498 case VMX_VMCS_GUEST_GS_BASE:
5499 case VMX_VMCS_GUEST_LDTR_BASE:
5500 case VMX_VMCS_GUEST_TR_BASE:
5501 case VMX_VMCS_GUEST_GDTR_BASE:
5502 case VMX_VMCS_GUEST_IDTR_BASE:
5503 case VMX_VMCS_GUEST_RSP:
5504 case VMX_VMCS_GUEST_RIP:
5505 case VMX_VMCS_GUEST_SYSENTER_ESP:
5506 case VMX_VMCS_GUEST_SYSENTER_EIP:
5507 {
5508 if (!(RT_HI_U32(u64Val)))
5509 {
5510 /* If this field is 64-bit, VT-x will zero out the top bits. */
5511 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5512 }
5513 else
5514 {
5515 /* Assert that only the 32->64 switcher case should ever come here. */
5516 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5517 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5518 }
5519 break;
5520 }
5521
5522 default:
5523 {
5524 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5525 rc = VERR_INVALID_PARAMETER;
5526 break;
5527 }
5528 }
5529 AssertRCReturn(rc, rc);
5530 return rc;
5531}
5532
5533
5534/**
5535 * Queue up a VMWRITE by using the VMCS write cache.
5536 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5537 *
5538 * @param pVCpu The cross context virtual CPU structure.
5539 * @param idxField The VMCS field encoding.
5540 * @param u64Val 16, 32 or 64-bit value.
5541 */
5542VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5543{
5544 AssertPtr(pVCpu);
5545 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5546
5547 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5548 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5549
5550 /* Make sure there are no duplicates. */
5551 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5552 {
5553 if (pCache->Write.aField[i] == idxField)
5554 {
5555 pCache->Write.aFieldVal[i] = u64Val;
5556 return VINF_SUCCESS;
5557 }
5558 }
5559
5560 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5561 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5562 pCache->Write.cValidEntries++;
5563 return VINF_SUCCESS;
5564}
5565#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5566
5567
5568/**
5569 * Sets up the usage of TSC-offsetting and updates the VMCS.
5570 *
5571 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5572 * VMX preemption timer.
5573 *
5574 * @returns VBox status code.
5575 * @param pVM The cross context VM structure.
5576 * @param pVCpu The cross context virtual CPU structure.
5577 *
5578 * @remarks No-long-jump zone!!!
5579 */
5580static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5581{
5582 int rc;
5583 bool fOffsettedTsc;
5584 bool fParavirtTsc;
5585 if (pVM->hm.s.vmx.fUsePreemptTimer)
5586 {
5587 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5588 &fOffsettedTsc, &fParavirtTsc);
5589
5590 /* Make sure the returned values have sane upper and lower boundaries. */
5591 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5592 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5593 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5594 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5595
5596 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5597 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5598 }
5599 else
5600 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5601
5602 /** @todo later optimize this to be done elsewhere and not before every
5603 * VM-entry. */
5604 if (fParavirtTsc)
5605 {
5606 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5607 information before every VM-entry, hence disable it for performance sake. */
5608#if 0
5609 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5610 AssertRC(rc);
5611#endif
5612 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5613 }
5614
5615 if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5616 {
5617 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5618 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5619
5620 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5621 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5622 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5623 }
5624 else
5625 {
5626 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5627 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5628 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5629 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5630 }
5631}
5632
5633
5634/**
5635 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5636 * VM-exit interruption info type.
5637 *
5638 * @returns The IEM exception flags.
5639 * @param uVector The event vector.
5640 * @param uVmxVectorType The VMX event type.
5641 *
5642 * @remarks This function currently only constructs flags required for
5643 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5644 * and CR2 aspects of an exception are not included).
5645 */
5646static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5647{
5648 uint32_t fIemXcptFlags;
5649 switch (uVmxVectorType)
5650 {
5651 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5652 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5653 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5654 break;
5655
5656 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5657 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5658 break;
5659
5660 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5661 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5662 break;
5663
5664 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5665 {
5666 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5667 if (uVector == X86_XCPT_BP)
5668 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5669 else if (uVector == X86_XCPT_OF)
5670 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5671 else
5672 {
5673 fIemXcptFlags = 0;
5674 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5675 }
5676 break;
5677 }
5678
5679 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5680 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5681 break;
5682
5683 default:
5684 fIemXcptFlags = 0;
5685 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5686 break;
5687 }
5688 return fIemXcptFlags;
5689}
5690
5691
5692/**
5693 * Sets an event as a pending event to be injected into the guest.
5694 *
5695 * @param pVCpu The cross context virtual CPU structure.
5696 * @param u32IntInfo The VM-entry interruption-information field.
5697 * @param cbInstr The VM-entry instruction length in bytes (for software
5698 * interrupts, exceptions and privileged software
5699 * exceptions).
5700 * @param u32ErrCode The VM-entry exception error code.
5701 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5702 * page-fault.
5703 *
5704 * @remarks Statistics counter assumes this is a guest event being injected or
5705 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5706 * always incremented.
5707 */
5708DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5709 RTGCUINTPTR GCPtrFaultAddress)
5710{
5711 Assert(!pVCpu->hm.s.Event.fPending);
5712 pVCpu->hm.s.Event.fPending = true;
5713 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5714 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5715 pVCpu->hm.s.Event.cbInstr = cbInstr;
5716 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5717}
5718
5719
5720/**
5721 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5722 *
5723 * @param pVCpu The cross context virtual CPU structure.
5724 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5725 * out-of-sync. Make sure to update the required fields
5726 * before using them.
5727 */
5728DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5729{
5730 NOREF(pMixedCtx);
5731 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5732 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5733 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5734 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5735}
5736
5737
5738/**
5739 * Handle a condition that occurred while delivering an event through the guest
5740 * IDT.
5741 *
5742 * @returns Strict VBox status code (i.e. informational status codes too).
5743 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5744 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5745 * to continue execution of the guest which will delivery the \#DF.
5746 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5747 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5748 *
5749 * @param pVCpu The cross context virtual CPU structure.
5750 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5751 * out-of-sync. Make sure to update the required fields
5752 * before using them.
5753 * @param pVmxTransient Pointer to the VMX transient structure.
5754 *
5755 * @remarks No-long-jump zone!!!
5756 */
5757static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5758{
5759 uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5760
5761 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5762 rc2 |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5763 AssertRCReturn(rc2, rc2);
5764
5765 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5766 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5767 {
5768 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5769 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5770
5771 /*
5772 * If the event was a software interrupt (generated with INT n) or a software exception
5773 * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
5774 * can handle the VM-exit and continue guest execution which will re-execute the
5775 * instruction rather than re-injecting the exception, as that can cause premature
5776 * trips to ring-3 before injection and involve TRPM which currently has no way of
5777 * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
5778 * the problem).
5779 */
5780 IEMXCPTRAISE enmRaise;
5781 IEMXCPTRAISEINFO fRaiseInfo;
5782 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5783 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5784 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5785 {
5786 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5787 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5788 }
5789 else if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5790 {
5791 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5792 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5793 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5794 /** @todo Make AssertMsgReturn as just AssertMsg later. */
5795 AssertMsgReturn(uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT,
5796 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5797 uExitVectorType), VERR_VMX_IPE_5);
5798
5799 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5800
5801 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5802 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
5803 {
5804 pVmxTransient->fVectoringPF = true;
5805 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5806 }
5807 }
5808 else
5809 {
5810 /*
5811 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5812 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5813 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
5814 */
5815 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5816 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5817 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5818 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5819 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5820 }
5821
5822 /*
5823 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5824 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5825 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5826 * subsequent VM-entry would fail.
5827 *
5828 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5829 */
5830 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5831 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5832 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
5833 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
5834 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5835 {
5836 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5837 }
5838
5839 switch (enmRaise)
5840 {
5841 case IEMXCPTRAISE_CURRENT_XCPT:
5842 {
5843 Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
5844 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
5845 Assert(rcStrict == VINF_SUCCESS);
5846 break;
5847 }
5848
5849 case IEMXCPTRAISE_PREV_EVENT:
5850 {
5851 uint32_t u32ErrCode;
5852 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5853 {
5854 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5855 AssertRCReturn(rc2, rc2);
5856 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5857 }
5858 else
5859 u32ErrCode = 0;
5860
5861 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
5862 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5863 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5864 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5865
5866 Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
5867 pVCpu->hm.s.Event.u32ErrCode));
5868 Assert(rcStrict == VINF_SUCCESS);
5869 break;
5870 }
5871
5872 case IEMXCPTRAISE_REEXEC_INSTR:
5873 Assert(rcStrict == VINF_SUCCESS);
5874 break;
5875
5876 case IEMXCPTRAISE_DOUBLE_FAULT:
5877 {
5878 /*
5879 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
5880 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
5881 */
5882 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
5883 {
5884 pVmxTransient->fVectoringDoublePF = true;
5885 Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
5886 pMixedCtx->cr2));
5887 rcStrict = VINF_SUCCESS;
5888 }
5889 else
5890 {
5891 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5892 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5893 Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
5894 uIdtVector, uExitVector));
5895 rcStrict = VINF_HM_DOUBLE_FAULT;
5896 }
5897 break;
5898 }
5899
5900 case IEMXCPTRAISE_TRIPLE_FAULT:
5901 {
5902 Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
5903 rcStrict = VINF_EM_RESET;
5904 break;
5905 }
5906
5907 case IEMXCPTRAISE_CPU_HANG:
5908 {
5909 Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
5910 rcStrict = VERR_EM_GUEST_CPU_HANG;
5911 break;
5912 }
5913
5914 default:
5915 {
5916 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
5917 rcStrict = VERR_VMX_IPE_2;
5918 break;
5919 }
5920 }
5921 }
5922 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5923 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5924 && uExitVector != X86_XCPT_DF
5925 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5926 {
5927 /*
5928 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5929 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5930 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5931 */
5932 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5933 {
5934 Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
5935 VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5936 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5937 }
5938 }
5939
5940 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
5941 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
5942 return rcStrict;
5943}
5944
5945
5946/**
5947 * Imports a guest segment register from the current VMCS into
5948 * the guest-CPU context.
5949 *
5950 * @returns VBox status code.
5951 * @param pVCpu The cross context virtual CPU structure.
5952 * @param idxSel Index of the selector in the VMCS.
5953 * @param idxLimit Index of the segment limit in the VMCS.
5954 * @param idxBase Index of the segment base in the VMCS.
5955 * @param idxAccess Index of the access rights of the segment in the VMCS.
5956 * @param pSelReg Pointer to the segment selector.
5957 *
5958 * @remarks No-long-jump zone!!!
5959 *
5960 * @remarks Never call this function directly!!! Use the
5961 * HMVMX_IMPORT_SREG() macro as that takes care
5962 * of whether to read from the VMCS cache or not.
5963 */
5964static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5965 PCPUMSELREG pSelReg)
5966{
5967 NOREF(pVCpu);
5968
5969 uint32_t u32Sel;
5970 uint32_t u32Limit;
5971 uint32_t u32Attr;
5972 uint64_t u64Base;
5973 int rc = VMXReadVmcs32(idxSel, &u32Sel);
5974 rc |= VMXReadVmcs32(idxLimit, &u32Limit);
5975 rc |= VMXReadVmcs32(idxAccess, &u32Attr);
5976 rc |= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
5977 AssertRCReturn(rc, rc);
5978
5979 pSelReg->Sel = (uint16_t)u32Sel;
5980 pSelReg->ValidSel = (uint16_t)u32Sel;
5981 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5982 pSelReg->u32Limit = u32Limit;
5983 pSelReg->u64Base = u64Base;
5984 pSelReg->Attr.u = u32Attr;
5985
5986 /*
5987 * If VT-x marks the segment as unusable, most other bits remain undefined:
5988 * - For CS the L, D and G bits have meaning.
5989 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
5990 * - For the remaining data segments no bits are defined.
5991 *
5992 * The present bit and the unusable bit has been observed to be set at the
5993 * same time (the selector was supposed to be invalid as we started executing
5994 * a V8086 interrupt in ring-0).
5995 *
5996 * What should be important for the rest of the VBox code, is that the P bit is
5997 * cleared. Some of the other VBox code recognizes the unusable bit, but
5998 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
5999 * safe side here, we'll strip off P and other bits we don't care about. If
6000 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6001 *
6002 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6003 */
6004 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6005 {
6006 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6007
6008 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6009 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6010 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6011
6012 Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
6013#ifdef DEBUG_bird
6014 AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
6015 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6016 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6017#endif
6018 }
6019 return VINF_SUCCESS;
6020}
6021
6022
6023/**
6024 * Imports the guest RIP from the VMCS back into the guest-CPU context.
6025 *
6026 * @returns VBox status code.
6027 * @param pVCpu The cross context virtual CPU structure.
6028 *
6029 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6030 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6031 * instead!!!
6032 */
6033DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6034{
6035 uint64_t u64Val;
6036 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6037 if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6038 {
6039 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6040 if (RT_SUCCESS(rc))
6041 {
6042 pCtx->rip = u64Val;
6043 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6044 }
6045 return rc;
6046 }
6047 return VINF_SUCCESS;
6048}
6049
6050
6051/**
6052 * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6053 *
6054 * @returns VBox status code.
6055 * @param pVCpu The cross context virtual CPU structure.
6056 *
6057 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6058 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6059 * instead!!!
6060 */
6061DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6062{
6063 uint32_t u32Val;
6064 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6065 if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6066 {
6067 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6068 if (RT_SUCCESS(rc))
6069 {
6070 pCtx->eflags.u32 = u32Val;
6071
6072 /* Restore eflags for real-on-v86-mode hack. */
6073 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6074 {
6075 pCtx->eflags.Bits.u1VM = 0;
6076 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6077 }
6078 }
6079 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6080 return rc;
6081 }
6082 return VINF_SUCCESS;
6083}
6084
6085
6086/**
6087 * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6088 * context.
6089 *
6090 * @returns VBox status code.
6091 * @param pVCpu The cross context virtual CPU structure.
6092 *
6093 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6094 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6095 * instead!!!
6096 */
6097DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6098{
6099 uint32_t u32Val;
6100 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6101 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32Val);
6102 if (RT_SUCCESS(rc))
6103 {
6104 /*
6105 * We additionally have a requirement to import RIP, RFLAGS depending on whether we
6106 * might need them in hmR0VmxEvaluatePendingEvent().
6107 */
6108 if (!u32Val)
6109 {
6110 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6111 {
6112 rc = hmR0VmxImportGuestRip(pVCpu);
6113 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6114 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6115 }
6116
6117 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6118 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6119 }
6120 else
6121 {
6122 rc = hmR0VmxImportGuestRip(pVCpu);
6123 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6124
6125 if (u32Val & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6126 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6127 {
6128 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6129 }
6130 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6131 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6132
6133 if (u32Val & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6134 {
6135 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6136 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6137 }
6138 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6139 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6140 }
6141 }
6142 return rc;
6143}
6144
6145
6146/**
6147 * Worker for VMXR0ImportStateOnDemand.
6148 *
6149 * @returns VBox status code.
6150 * @param pVCpu The cross context virtual CPU structure.
6151 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6152 */
6153static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6154{
6155#define VMXLOCAL_BREAK_RC(a_rc) \
6156 if (RT_FAILURE(a_rc)) \
6157 break
6158
6159 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6160
6161 int rc = VINF_SUCCESS;
6162 PVM pVM = pVCpu->CTX_SUFF(pVM);
6163 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6164 uint64_t u64Val;
6165 uint32_t u32Val;
6166
6167 Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6168
6169 /*
6170 * We disable interrupts to make the updating of the state and in particular
6171 * the fExtrn modification atomic wrt to preemption hooks.
6172 */
6173 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6174
6175 fWhat &= pCtx->fExtrn;
6176 if (fWhat & pCtx->fExtrn)
6177 {
6178 do
6179 {
6180 if (fWhat & CPUMCTX_EXTRN_RIP)
6181 {
6182 rc = hmR0VmxImportGuestRip(pVCpu);
6183 VMXLOCAL_BREAK_RC(rc);
6184 }
6185
6186 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6187 {
6188 rc = hmR0VmxImportGuestRFlags(pVCpu);
6189 VMXLOCAL_BREAK_RC(rc);
6190 }
6191
6192 if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6193 {
6194 rc = hmR0VmxImportGuestIntrState(pVCpu);
6195 VMXLOCAL_BREAK_RC(rc);
6196 }
6197
6198 if (fWhat & CPUMCTX_EXTRN_RSP)
6199 {
6200 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6201 VMXLOCAL_BREAK_RC(rc);
6202 pCtx->rsp = u64Val;
6203 }
6204
6205 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6206 {
6207 if (fWhat & CPUMCTX_EXTRN_CS)
6208 {
6209 rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6210 VMXLOCAL_BREAK_RC(rc);
6211 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6212 pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6213 }
6214 if (fWhat & CPUMCTX_EXTRN_SS)
6215 {
6216 rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6217 VMXLOCAL_BREAK_RC(rc);
6218 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6219 pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6220 }
6221 if (fWhat & CPUMCTX_EXTRN_DS)
6222 {
6223 rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6224 VMXLOCAL_BREAK_RC(rc);
6225 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6226 pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6227 }
6228 if (fWhat & CPUMCTX_EXTRN_ES)
6229 {
6230 rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6231 VMXLOCAL_BREAK_RC(rc);
6232 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6233 pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6234 }
6235 if (fWhat & CPUMCTX_EXTRN_FS)
6236 {
6237 rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6238 VMXLOCAL_BREAK_RC(rc);
6239 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6240 pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6241 }
6242 if (fWhat & CPUMCTX_EXTRN_GS)
6243 {
6244 rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6245 VMXLOCAL_BREAK_RC(rc);
6246 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6247 pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6248 }
6249 }
6250
6251 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6252 {
6253 if (fWhat & CPUMCTX_EXTRN_LDTR)
6254 {
6255 rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6256 VMXLOCAL_BREAK_RC(rc);
6257 }
6258
6259 if (fWhat & CPUMCTX_EXTRN_GDTR)
6260 {
6261 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6262 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6263 VMXLOCAL_BREAK_RC(rc);
6264 pCtx->gdtr.pGdt = u64Val;
6265 pCtx->gdtr.cbGdt = u32Val;
6266 }
6267
6268 /* Guest IDTR. */
6269 if (fWhat & CPUMCTX_EXTRN_IDTR)
6270 {
6271 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6272 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6273 VMXLOCAL_BREAK_RC(rc);
6274 pCtx->idtr.pIdt = u64Val;
6275 pCtx->idtr.cbIdt = u32Val;
6276 }
6277
6278 /* Guest TR. */
6279 if (fWhat & CPUMCTX_EXTRN_TR)
6280 {
6281 /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6282 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6283 {
6284 rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6285 VMXLOCAL_BREAK_RC(rc);
6286 }
6287 }
6288 }
6289
6290 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6291 {
6292 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6293 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6294 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6295 pCtx->SysEnter.cs = u32Val;
6296 VMXLOCAL_BREAK_RC(rc);
6297 }
6298
6299#if HC_ARCH_BITS == 64
6300 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6301 {
6302 if ( pVM->hm.s.fAllow64BitGuests
6303 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6304 pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6305 }
6306
6307 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6308 {
6309 if ( pVM->hm.s.fAllow64BitGuests
6310 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6311 {
6312 pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6313 pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6314 pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6315 }
6316 }
6317#endif
6318
6319 if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
6320#if HC_ARCH_BITS == 32
6321 || (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS))
6322#endif
6323 )
6324 {
6325 PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6326 uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6327 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6328 {
6329 switch (pMsr->u32Msr)
6330 {
6331#if HC_ARCH_BITS == 32
6332 case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6333 case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6334 case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6335 case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6336#endif
6337 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6338 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6339 default:
6340 {
6341 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6342 cMsrs));
6343 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6344 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6345 }
6346 }
6347 }
6348 }
6349
6350 if (fWhat & CPUMCTX_EXTRN_DR7)
6351 {
6352 if (!pVCpu->hm.s.fUsingHyperDR7)
6353 {
6354 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6355 rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6356 VMXLOCAL_BREAK_RC(rc);
6357 pCtx->dr[7] = u32Val;
6358 }
6359 }
6360
6361 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6362 {
6363 uint32_t u32Shadow;
6364 /* CR0 required for saving CR3 below, see CPUMIsGuestPagingEnabledEx(). */
6365 if (fWhat & CPUMCTX_EXTRN_CR0)
6366 {
6367 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6368 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6369 VMXLOCAL_BREAK_RC(rc);
6370 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32CR0Mask)
6371 | (u32Shadow & pVCpu->hm.s.vmx.u32CR0Mask);
6372 VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6373 CPUMSetGuestCR0(pVCpu, u32Val);
6374 VMMRZCallRing3Enable(pVCpu);
6375 }
6376
6377 /* CR4 required for saving CR3 below, see CPUMIsGuestInPAEModeEx(). */
6378 if (fWhat & CPUMCTX_EXTRN_CR4)
6379 {
6380 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6381 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6382 VMXLOCAL_BREAK_RC(rc);
6383 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32CR4Mask)
6384 | (u32Shadow & pVCpu->hm.s.vmx.u32CR4Mask);
6385 CPUMSetGuestCR4(pVCpu, u32Val);
6386 }
6387
6388 if (fWhat & CPUMCTX_EXTRN_CR3)
6389 {
6390 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6391 || ( pVM->hm.s.fNestedPaging
6392 && CPUMIsGuestPagingEnabledEx(pCtx))) /* PG bit changes are always intercepted, so it's up to date. */
6393 {
6394 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6395 if (pCtx->cr3 != u64Val)
6396 {
6397 CPUMSetGuestCR3(pVCpu, u64Val);
6398 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6399 }
6400
6401 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6402 if (CPUMIsGuestInPAEModeEx(pCtx))
6403 {
6404 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6405 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6406 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6407 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6408 VMXLOCAL_BREAK_RC(rc);
6409 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6410 }
6411 }
6412 }
6413 }
6414 } while (0);
6415
6416 if (RT_SUCCESS(rc))
6417 {
6418 /* Update fExtrn. */
6419 pCtx->fExtrn &= ~fWhat;
6420
6421 /* If everything has been imported, clear the HM keeper bit. */
6422 if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6423 {
6424 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6425 Assert(!pCtx->fExtrn);
6426 }
6427 }
6428 }
6429 else
6430 AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6431
6432 ASMSetFlags(fEFlags);
6433
6434 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6435
6436 /*
6437 * Honor any pending CR3 updates.
6438 *
6439 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6440 * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6441 * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6442 *
6443 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6444 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6445 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6446 * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6447 *
6448 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6449 */
6450 if (VMMRZCallRing3IsEnabled(pVCpu))
6451 {
6452 VMMR0LogFlushDisable(pVCpu);
6453
6454 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6455 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6456
6457 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6458 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6459
6460 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6461 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6462
6463 VMMR0LogFlushEnable(pVCpu);
6464 }
6465
6466 return VINF_SUCCESS;
6467#undef VMXLOCAL_BREAK_RC
6468}
6469
6470
6471/**
6472 * Saves the guest state from the VMCS into the guest-CPU context.
6473 *
6474 * @returns VBox status code.
6475 * @param pVCpu The cross context virtual CPU structure.
6476 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6477 */
6478VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6479{
6480 return hmR0VmxImportGuestState(pVCpu, fWhat);
6481}
6482
6483
6484/**
6485 * Check per-VM and per-VCPU force flag actions that require us to go back to
6486 * ring-3 for one reason or another.
6487 *
6488 * @returns Strict VBox status code (i.e. informational status codes too)
6489 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6490 * ring-3.
6491 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6492 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6493 * interrupts)
6494 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6495 * all EMTs to be in ring-3.
6496 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6497 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6498 * to the EM loop.
6499 *
6500 * @param pVM The cross context VM structure.
6501 * @param pVCpu The cross context virtual CPU structure.
6502 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6503 * out-of-sync. Make sure to update the required fields
6504 * before using them.
6505 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6506 */
6507static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6508{
6509 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6510
6511 /*
6512 * Anything pending? Should be more likely than not if we're doing a good job.
6513 */
6514 if ( !fStepping
6515 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6516 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6517 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6518 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6519 return VINF_SUCCESS;
6520
6521#if 0
6522 /* We need the control registers now, make sure the guest-CPU context is updated. */
6523 int rc3 = hmR0VmxImportGuestStatae(pVCpu, CPUMCTX_EXTRN_CR0);
6524 AssertRCReturn(rc3, rc3);
6525
6526 /** @todo r=ramshankar: VMCPU_FF_HM_UPDATE_CR3 and VMCPU_FF_HM_UPDATE_PAE_PDPES
6527 * are not part of VMCPU_FF_HP_R0_PRE_HM_MASK. Hence, the two if
6528 * statements below won't ever be entered. Consider removing it or
6529 * determine if it is necessary to add these flags to VMCPU_FF_HP_R0_PRE_HM_MASK. */
6530 /* Pending HM CR3 sync. */
6531 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6532 {
6533 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6534 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6535 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6536 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6537 }
6538
6539 /* Pending HM PAE PDPEs. */
6540 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6541 {
6542 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6543 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6544 }
6545#endif
6546
6547 /* Pending PGM C3 sync. */
6548 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6549 {
6550 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
6551 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6552 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6553 if (rcStrict2 != VINF_SUCCESS)
6554 {
6555 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6556 Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6557 return rcStrict2;
6558 }
6559 }
6560
6561 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6562 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6563 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6564 {
6565 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6566 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6567 Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6568 return rc2;
6569 }
6570
6571 /* Pending VM request packets, such as hardware interrupts. */
6572 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6573 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6574 {
6575 Log4Func(("Pending VM request forcing us back to ring-3\n"));
6576 return VINF_EM_PENDING_REQUEST;
6577 }
6578
6579 /* Pending PGM pool flushes. */
6580 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6581 {
6582 Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6583 return VINF_PGM_POOL_FLUSH_PENDING;
6584 }
6585
6586 /* Pending DMA requests. */
6587 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6588 {
6589 Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6590 return VINF_EM_RAW_TO_R3;
6591 }
6592
6593 return VINF_SUCCESS;
6594}
6595
6596
6597/**
6598 * Converts any TRPM trap into a pending HM event. This is typically used when
6599 * entering from ring-3 (not longjmp returns).
6600 *
6601 * @param pVCpu The cross context virtual CPU structure.
6602 */
6603static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6604{
6605 Assert(TRPMHasTrap(pVCpu));
6606 Assert(!pVCpu->hm.s.Event.fPending);
6607
6608 uint8_t uVector;
6609 TRPMEVENT enmTrpmEvent;
6610 RTGCUINT uErrCode;
6611 RTGCUINTPTR GCPtrFaultAddress;
6612 uint8_t cbInstr;
6613
6614 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6615 AssertRC(rc);
6616
6617 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6618 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6619 if (enmTrpmEvent == TRPM_TRAP)
6620 {
6621 switch (uVector)
6622 {
6623 case X86_XCPT_NMI:
6624 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6625 break;
6626
6627 case X86_XCPT_BP:
6628 case X86_XCPT_OF:
6629 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6630 break;
6631
6632 case X86_XCPT_PF:
6633 case X86_XCPT_DF:
6634 case X86_XCPT_TS:
6635 case X86_XCPT_NP:
6636 case X86_XCPT_SS:
6637 case X86_XCPT_GP:
6638 case X86_XCPT_AC:
6639 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6640 RT_FALL_THRU();
6641 default:
6642 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6643 break;
6644 }
6645 }
6646 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6647 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6648 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6649 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6650 else
6651 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6652
6653 rc = TRPMResetTrap(pVCpu);
6654 AssertRC(rc);
6655 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6656 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6657
6658 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6659}
6660
6661
6662/**
6663 * Converts the pending HM event into a TRPM trap.
6664 *
6665 * @param pVCpu The cross context virtual CPU structure.
6666 */
6667static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6668{
6669 Assert(pVCpu->hm.s.Event.fPending);
6670
6671 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6672 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6673 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6674 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6675
6676 /* If a trap was already pending, we did something wrong! */
6677 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6678
6679 TRPMEVENT enmTrapType;
6680 switch (uVectorType)
6681 {
6682 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6683 enmTrapType = TRPM_HARDWARE_INT;
6684 break;
6685
6686 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6687 enmTrapType = TRPM_SOFTWARE_INT;
6688 break;
6689
6690 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6691 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6692 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6693 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6694 enmTrapType = TRPM_TRAP;
6695 break;
6696
6697 default:
6698 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6699 enmTrapType = TRPM_32BIT_HACK;
6700 break;
6701 }
6702
6703 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6704
6705 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6706 AssertRC(rc);
6707
6708 if (fErrorCodeValid)
6709 TRPMSetErrorCode(pVCpu, uErrorCode);
6710
6711 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6712 && uVector == X86_XCPT_PF)
6713 {
6714 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6715 }
6716 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6717 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6718 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6719 {
6720 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6721 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6722 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6723 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6724 }
6725
6726 /* Clear any pending events from the VMCS. */
6727 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
6728 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
6729
6730 /* We're now done converting the pending event. */
6731 pVCpu->hm.s.Event.fPending = false;
6732}
6733
6734
6735/**
6736 * Does the necessary state syncing before returning to ring-3 for any reason
6737 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6738 *
6739 * @returns VBox status code.
6740 * @param pVCpu The cross context virtual CPU structure.
6741 * @param fImportState Whether to import the guest state from the VMCS back
6742 * to the guest-CPU context.
6743 *
6744 * @remarks No-long-jmp zone!!!
6745 */
6746static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
6747{
6748 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6749 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6750
6751 RTCPUID idCpu = RTMpCpuId();
6752 Log4Func(("HostCpuId=%u\n", idCpu));
6753
6754 /*
6755 * !!! IMPORTANT !!!
6756 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
6757 */
6758
6759 /* Save the guest state if necessary. */
6760 if (fImportState)
6761 {
6762 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
6763 AssertRCReturn(rc, rc);
6764 }
6765
6766 /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
6767 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
6768 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6769
6770 /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
6771#ifdef VBOX_STRICT
6772 if (CPUMIsHyperDebugStateActive(pVCpu))
6773 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6774#endif
6775 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6776 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6777 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6778
6779#if HC_ARCH_BITS == 64
6780 /* Restore host-state bits that VT-x only restores partially. */
6781 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6782 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6783 {
6784 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6785 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6786 }
6787 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6788#endif
6789
6790 /* Restore the lazy host MSRs as we're leaving VT-x context. */
6791 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
6792 {
6793 /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
6794 if (!fImportState)
6795 {
6796 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE
6797 | CPUMCTX_EXTRN_SYSCALL_MSRS);
6798 AssertRCReturn(rc, rc);
6799 }
6800 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6801 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
6802 }
6803 else
6804 pVCpu->hm.s.vmx.fLazyMsrs = 0;
6805
6806 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6807 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6808
6809 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6810 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
6811 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
6812 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6813 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6814 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6815 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6816 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6817 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6818
6819 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6820
6821 /** @todo This partially defeats the purpose of having preemption hooks.
6822 * The problem is, deregistering the hooks should be moved to a place that
6823 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6824 * context.
6825 */
6826 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6827 {
6828 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6829 AssertRCReturn(rc, rc);
6830
6831 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6832 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6833 }
6834 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6835 NOREF(idCpu);
6836
6837 return VINF_SUCCESS;
6838}
6839
6840
6841/**
6842 * Leaves the VT-x session.
6843 *
6844 * @returns VBox status code.
6845 * @param pVCpu The cross context virtual CPU structure.
6846 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6847 * out-of-sync. Make sure to update the required fields
6848 * before using them.
6849 *
6850 * @remarks No-long-jmp zone!!!
6851 */
6852static int hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6853{
6854 HM_DISABLE_PREEMPT();
6855 HMVMX_ASSERT_CPU_SAFE();
6856 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6857 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6858
6859 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6860 and done this from the VMXR0ThreadCtxCallback(). */
6861 if (!pVCpu->hm.s.fLeaveDone)
6862 {
6863 int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
6864 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
6865 pVCpu->hm.s.fLeaveDone = true;
6866 }
6867 Assert(!pMixedCtx->fExtrn);
6868
6869 /*
6870 * !!! IMPORTANT !!!
6871 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6872 */
6873
6874 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6875 /** @todo Deregistering here means we need to VMCLEAR always
6876 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6877 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
6878 VMMR0ThreadCtxHookDisable(pVCpu);
6879
6880 /* Leave HM context. This takes care of local init (term). */
6881 int rc = HMR0LeaveCpu(pVCpu);
6882
6883 HM_RESTORE_PREEMPT();
6884 return rc;
6885}
6886
6887
6888/**
6889 * Does the necessary state syncing before doing a longjmp to ring-3.
6890 *
6891 * @returns VBox status code.
6892 * @param pVCpu The cross context virtual CPU structure.
6893 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6894 * out-of-sync. Make sure to update the required fields
6895 * before using them.
6896 *
6897 * @remarks No-long-jmp zone!!!
6898 */
6899DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6900{
6901 return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
6902}
6903
6904
6905/**
6906 * Take necessary actions before going back to ring-3.
6907 *
6908 * An action requires us to go back to ring-3. This function does the necessary
6909 * steps before we can safely return to ring-3. This is not the same as longjmps
6910 * to ring-3, this is voluntary and prepares the guest so it may continue
6911 * executing outside HM (recompiler/IEM).
6912 *
6913 * @returns VBox status code.
6914 * @param pVM The cross context VM structure.
6915 * @param pVCpu The cross context virtual CPU structure.
6916 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6917 * out-of-sync. Make sure to update the required fields
6918 * before using them.
6919 * @param rcExit The reason for exiting to ring-3. Can be
6920 * VINF_VMM_UNKNOWN_RING3_CALL.
6921 */
6922static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
6923{
6924 Assert(pVM);
6925 Assert(pVCpu);
6926 Assert(pMixedCtx);
6927 HMVMX_ASSERT_PREEMPT_SAFE();
6928
6929 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6930 {
6931 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6932 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6933 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6934 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6935 }
6936
6937 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6938 VMMRZCallRing3Disable(pVCpu);
6939 Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
6940
6941 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6942 if (pVCpu->hm.s.Event.fPending)
6943 {
6944 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6945 Assert(!pVCpu->hm.s.Event.fPending);
6946 }
6947
6948 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
6949 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
6950
6951 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
6952 and if we're injecting an event we should have a TRPM trap pending. */
6953 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6954#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
6955 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6956#endif
6957
6958 /* Save guest state and restore host state bits. */
6959 int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
6960 AssertRCReturn(rc, rc);
6961 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6962 /* Thread-context hooks are unregistered at this point!!! */
6963
6964 /* Sync recompiler state. */
6965 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6966 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6967 | CPUM_CHANGED_LDTR
6968 | CPUM_CHANGED_GDTR
6969 | CPUM_CHANGED_IDTR
6970 | CPUM_CHANGED_TR
6971 | CPUM_CHANGED_HIDDEN_SEL_REGS);
6972 if ( pVM->hm.s.fNestedPaging
6973 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
6974 {
6975 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6976 }
6977
6978 Assert(!pVCpu->hm.s.fClearTrapFlag);
6979
6980 /* Update the exit-to-ring 3 reason. */
6981 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
6982
6983 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6984 if (rcExit != VINF_EM_RAW_INTERRUPT)
6985 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
6986
6987 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6988
6989 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6990 VMMRZCallRing3RemoveNotification(pVCpu);
6991 VMMRZCallRing3Enable(pVCpu);
6992
6993 return rc;
6994}
6995
6996
6997/**
6998 * VMMRZCallRing3() callback wrapper which saves the guest state before we
6999 * longjump to ring-3 and possibly get preempted.
7000 *
7001 * @returns VBox status code.
7002 * @param pVCpu The cross context virtual CPU structure.
7003 * @param enmOperation The operation causing the ring-3 longjump.
7004 * @param pvUser Opaque pointer to the guest-CPU context. The data
7005 * may be out-of-sync. Make sure to update the required
7006 * fields before using them.
7007 */
7008static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7009{
7010 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7011 {
7012 /*
7013 * !!! IMPORTANT !!!
7014 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7015 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7016 */
7017 VMMRZCallRing3RemoveNotification(pVCpu);
7018 VMMRZCallRing3Disable(pVCpu);
7019 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7020 RTThreadPreemptDisable(&PreemptState);
7021
7022 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7023 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7024
7025#if HC_ARCH_BITS == 64
7026 /* Restore host-state bits that VT-x only restores partially. */
7027 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7028 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7029 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7030 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7031#endif
7032
7033 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7034 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7035 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7036
7037 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7038 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7039 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7040 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7041 {
7042 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7043 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7044 }
7045
7046 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7047 VMMR0ThreadCtxHookDisable(pVCpu);
7048 HMR0LeaveCpu(pVCpu);
7049 RTThreadPreemptRestore(&PreemptState);
7050 return VINF_SUCCESS;
7051 }
7052
7053 Assert(pVCpu);
7054 Assert(pvUser);
7055 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7056 HMVMX_ASSERT_PREEMPT_SAFE();
7057
7058 VMMRZCallRing3Disable(pVCpu);
7059 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7060
7061 Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7062
7063 int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7064 AssertRCReturn(rc, rc);
7065
7066 VMMRZCallRing3Enable(pVCpu);
7067 return VINF_SUCCESS;
7068}
7069
7070
7071/**
7072 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7073 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7074 *
7075 * @param pVCpu The cross context virtual CPU structure.
7076 */
7077DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7078{
7079 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7080 {
7081 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7082 {
7083 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7084 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7085 AssertRC(rc);
7086 Log4Func(("Setup interrupt-window exiting\n"));
7087 }
7088 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7089}
7090
7091
7092/**
7093 * Clears the interrupt-window exiting control in the VMCS.
7094 *
7095 * @param pVCpu The cross context virtual CPU structure.
7096 */
7097DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7098{
7099 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7100 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7101 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7102 AssertRC(rc);
7103 Log4Func(("Cleared interrupt-window exiting\n"));
7104}
7105
7106
7107/**
7108 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7109 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7110 *
7111 * @param pVCpu The cross context virtual CPU structure.
7112 */
7113DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7114{
7115 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7116 {
7117 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7118 {
7119 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7120 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7121 AssertRC(rc);
7122 Log4Func(("Setup NMI-window exiting\n"));
7123 }
7124 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7125}
7126
7127
7128/**
7129 * Clears the NMI-window exiting control in the VMCS.
7130 *
7131 * @param pVCpu The cross context virtual CPU structure.
7132 */
7133DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7134{
7135 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7136 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7137 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7138 AssertRC(rc);
7139 Log4Func(("Cleared NMI-window exiting\n"));
7140}
7141
7142
7143/**
7144 * Evaluates the event to be delivered to the guest and sets it as the pending
7145 * event.
7146 *
7147 * @returns The VT-x guest-interruptibility state.
7148 * @param pVCpu The cross context virtual CPU structure.
7149 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7150 * out-of-sync. Make sure to update the required fields
7151 * before using them.
7152 */
7153static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7154{
7155 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7156 uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7157 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7158 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7159 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7160
7161 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7162 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7163 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7164 Assert(!TRPMHasTrap(pVCpu));
7165
7166 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7167 APICUpdatePendingInterrupts(pVCpu);
7168
7169 /*
7170 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7171 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7172 */
7173 /** @todo SMI. SMIs take priority over NMIs. */
7174 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7175 {
7176 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7177 if ( !pVCpu->hm.s.Event.fPending
7178 && !fBlockNmi
7179 && !fBlockSti
7180 && !fBlockMovSS)
7181 {
7182 Log4Func(("Pending NMI\n"));
7183 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7184 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7185
7186 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7187 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7188 }
7189 else
7190 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7191 }
7192 /*
7193 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7194 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7195 */
7196 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7197 && !pVCpu->hm.s.fSingleInstruction)
7198 {
7199 Assert(!DBGFIsStepping(pVCpu));
7200 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7201 AssertRCReturn(rc, 0);
7202 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7203 if ( !pVCpu->hm.s.Event.fPending
7204 && !fBlockInt
7205 && !fBlockSti
7206 && !fBlockMovSS)
7207 {
7208 uint8_t u8Interrupt;
7209 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7210 if (RT_SUCCESS(rc))
7211 {
7212 Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7213 uint32_t u32IntInfo = u8Interrupt
7214 | VMX_EXIT_INTERRUPTION_INFO_VALID
7215 | (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7216
7217 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7218 }
7219 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7220 {
7221 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7222 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7223 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7224
7225 /*
7226 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7227 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7228 * need to re-set this force-flag here.
7229 */
7230 }
7231 else
7232 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7233 }
7234 else
7235 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7236 }
7237
7238 return fIntrState;
7239}
7240
7241
7242/**
7243 * Sets a pending-debug exception to be delivered to the guest if the guest is
7244 * single-stepping in the VMCS.
7245 *
7246 * @param pVCpu The cross context virtual CPU structure.
7247 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7248 * out-of-sync. Make sure to update the required fields
7249 * before using them.
7250 */
7251DECLINLINE(int) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7252{
7253 RT_NOREF(pVCpu);
7254 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS)); NOREF(pMixedCtx);
7255 return VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7256}
7257
7258
7259/**
7260 * Injects any pending events into the guest if the guest is in a state to
7261 * receive them.
7262 *
7263 * @returns Strict VBox status code (i.e. informational status codes too).
7264 * @param pVCpu The cross context virtual CPU structure.
7265 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7266 * out-of-sync. Make sure to update the required fields
7267 * before using them.
7268 * @param fIntrState The VT-x guest-interruptibility state.
7269 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7270 * return VINF_EM_DBG_STEPPED if the event was
7271 * dispatched directly.
7272 */
7273static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t fIntrState, bool fStepping)
7274{
7275 HMVMX_ASSERT_PREEMPT_SAFE();
7276 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7277
7278 bool fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7279 bool fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7280
7281 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7282 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7283 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7284 Assert(!TRPMHasTrap(pVCpu));
7285
7286 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7287 if (pVCpu->hm.s.Event.fPending)
7288 {
7289 /*
7290 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7291 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7292 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7293 *
7294 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7295 */
7296 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7297#ifdef VBOX_STRICT
7298 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7299 {
7300 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7301 Assert(!fBlockInt);
7302 Assert(!fBlockSti);
7303 Assert(!fBlockMovSS);
7304 }
7305 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7306 {
7307 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7308 Assert(!fBlockSti);
7309 Assert(!fBlockMovSS);
7310 Assert(!fBlockNmi);
7311 }
7312#endif
7313 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7314 (uint8_t)uIntType));
7315 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7316 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7317 &fIntrState);
7318 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7319
7320 /* Update the interruptibility-state as it could have been changed by
7321 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7322 fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7323 fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7324
7325 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7326 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7327 else
7328 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7329 }
7330
7331 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7332 if ( fBlockSti
7333 || fBlockMovSS)
7334 {
7335 if (!pVCpu->hm.s.fSingleInstruction)
7336 {
7337 /*
7338 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7339 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7340 * See Intel spec. 27.3.4 "Saving Non-Register State".
7341 */
7342 Assert(!DBGFIsStepping(pVCpu));
7343 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7344 AssertRCReturn(rc, rc);
7345 if (pMixedCtx->eflags.Bits.u1TF)
7346 {
7347 int rc2 = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
7348 AssertRCReturn(rc2, rc2);
7349 }
7350 }
7351 else if (pMixedCtx->eflags.Bits.u1TF)
7352 {
7353 /*
7354 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7355 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7356 */
7357 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7358 fIntrState = 0;
7359 }
7360 }
7361
7362 /*
7363 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7364 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7365 */
7366 int rc3 = hmR0VmxExportGuestIntrState(pVCpu, fIntrState);
7367 AssertRCReturn(rc3, rc3);
7368
7369 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7370 NOREF(fBlockMovSS); NOREF(fBlockSti);
7371 return rcStrict;
7372}
7373
7374
7375/**
7376 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7377 *
7378 * @param pVCpu The cross context virtual CPU structure.
7379 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7380 * out-of-sync. Make sure to update the required fields
7381 * before using them.
7382 */
7383DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7384{
7385 NOREF(pMixedCtx);
7386 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7387 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7388}
7389
7390
7391/**
7392 * Injects a double-fault (\#DF) exception into the VM.
7393 *
7394 * @returns Strict VBox status code (i.e. informational status codes too).
7395 * @param pVCpu The cross context virtual CPU structure.
7396 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7397 * out-of-sync. Make sure to update the required fields
7398 * before using them.
7399 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7400 * and should return VINF_EM_DBG_STEPPED if the event
7401 * is injected directly (register modified by us, not
7402 * by hardware on VM-entry).
7403 * @param pfIntrState Pointer to the current guest interruptibility-state.
7404 * This interruptibility-state will be updated if
7405 * necessary. This cannot not be NULL.
7406 */
7407DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCCPUMCTX pMixedCtx, bool fStepping, uint32_t *pfIntrState)
7408{
7409 NOREF(pMixedCtx);
7410 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7411 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7412 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7413 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */, fStepping,
7414 pfIntrState);
7415}
7416
7417
7418/**
7419 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7420 *
7421 * @param pVCpu The cross context virtual CPU structure.
7422 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7423 * out-of-sync. Make sure to update the required fields
7424 * before using them.
7425 */
7426DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7427{
7428 NOREF(pMixedCtx);
7429 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7430 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7431 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7432}
7433
7434
7435/**
7436 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7437 *
7438 * @param pVCpu The cross context virtual CPU structure.
7439 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7440 * out-of-sync. Make sure to update the required fields
7441 * before using them.
7442 * @param cbInstr The value of RIP that is to be pushed on the guest
7443 * stack.
7444 */
7445DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7446{
7447 NOREF(pMixedCtx);
7448 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7449 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7450 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7451}
7452
7453
7454/**
7455 * Injects a general-protection (\#GP) fault into the VM.
7456 *
7457 * @returns Strict VBox status code (i.e. informational status codes too).
7458 * @param pVCpu The cross context virtual CPU structure.
7459 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7460 * out-of-sync. Make sure to update the required fields
7461 * before using them.
7462 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7463 * mode, i.e. in real-mode it's not valid).
7464 * @param u32ErrorCode The error code associated with the \#GP.
7465 * @param fStepping Whether we're running in
7466 * hmR0VmxRunGuestCodeStep() and should return
7467 * VINF_EM_DBG_STEPPED if the event is injected
7468 * directly (register modified by us, not by
7469 * hardware on VM-entry).
7470 * @param pfIntrState Pointer to the current guest interruptibility-state.
7471 * This interruptibility-state will be updated if
7472 * necessary. This cannot not be NULL.
7473 */
7474DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7475 bool fStepping, uint32_t *pfIntrState)
7476{
7477 NOREF(pMixedCtx);
7478 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7479 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7480 if (fErrorCodeValid)
7481 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7482 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */, fStepping,
7483 pfIntrState);
7484}
7485
7486
7487#if 0 /* unused */
7488/**
7489 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7490 * VM.
7491 *
7492 * @param pVCpu The cross context virtual CPU structure.
7493 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7494 * out-of-sync. Make sure to update the required fields
7495 * before using them.
7496 * @param u32ErrorCode The error code associated with the \#GP.
7497 */
7498DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7499{
7500 NOREF(pMixedCtx);
7501 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7502 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7503 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7504 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7505}
7506#endif /* unused */
7507
7508
7509/**
7510 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7511 *
7512 * @param pVCpu The cross context virtual CPU structure.
7513 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7514 * out-of-sync. Make sure to update the required fields
7515 * before using them.
7516 * @param uVector The software interrupt vector number.
7517 * @param cbInstr The value of RIP that is to be pushed on the guest
7518 * stack.
7519 */
7520DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7521{
7522 NOREF(pMixedCtx);
7523 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7524 if ( uVector == X86_XCPT_BP
7525 || uVector == X86_XCPT_OF)
7526 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7527 else
7528 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7529 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7530}
7531
7532
7533/**
7534 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7535 * stack.
7536 *
7537 * @returns Strict VBox status code (i.e. informational status codes too).
7538 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7539 * @param pVM The cross context VM structure.
7540 * @param pMixedCtx Pointer to the guest-CPU context.
7541 * @param uValue The value to push to the guest stack.
7542 */
7543DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7544{
7545 /*
7546 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7547 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7548 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7549 */
7550 if (pMixedCtx->sp == 1)
7551 return VINF_EM_RESET;
7552 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7553 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7554 AssertRC(rc);
7555 return rc;
7556}
7557
7558
7559/**
7560 * Injects an event into the guest upon VM-entry by updating the relevant fields
7561 * in the VM-entry area in the VMCS.
7562 *
7563 * @returns Strict VBox status code (i.e. informational status codes too).
7564 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7565 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7566 *
7567 * @param pVCpu The cross context virtual CPU structure.
7568 * @param u64IntInfo The VM-entry interruption-information field.
7569 * @param cbInstr The VM-entry instruction length in bytes (for
7570 * software interrupts, exceptions and privileged
7571 * software exceptions).
7572 * @param u32ErrCode The VM-entry exception error code.
7573 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7574 * @param pfIntrState Pointer to the current guest interruptibility-state.
7575 * This interruptibility-state will be updated if
7576 * necessary. This cannot not be NULL.
7577 * @param fStepping Whether we're running in
7578 * hmR0VmxRunGuestCodeStep() and should return
7579 * VINF_EM_DBG_STEPPED if the event is injected
7580 * directly (register modified by us, not by
7581 * hardware on VM-entry).
7582 *
7583 * @remarks Requires CR0!
7584 */
7585static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7586 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7587{
7588 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7589 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7590 Assert(pfIntrState);
7591
7592 PCPUMCTX pMixedCtx = &pVCpu->cpum.GstCtx;
7593 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7594 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7595 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7596
7597#ifdef VBOX_STRICT
7598 /*
7599 * Validate the error-code-valid bit for hardware exceptions.
7600 * No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7601 */
7602 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7603 && !CPUMIsGuestInRealModeEx(pMixedCtx))
7604 {
7605 switch (uVector)
7606 {
7607 case X86_XCPT_PF:
7608 case X86_XCPT_DF:
7609 case X86_XCPT_TS:
7610 case X86_XCPT_NP:
7611 case X86_XCPT_SS:
7612 case X86_XCPT_GP:
7613 case X86_XCPT_AC:
7614 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7615 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7616 RT_FALL_THRU();
7617 default:
7618 break;
7619 }
7620 }
7621#endif
7622
7623 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7624 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7625 || !(*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7626
7627 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7628
7629 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7630 {
7631 /*
7632 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7633 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7634 */
7635 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7636 }
7637 else
7638 {
7639 /* We require CR0 to check if the guest is in real-mode. */
7640 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
7641 AssertRCReturn(rc, rc);
7642
7643 /*
7644 * Hardware interrupts & exceptions cannot be delivered through the software interrupt
7645 * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7646 * interrupt handler in the (real-mode) guest.
7647 *
7648 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7649 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7650 */
7651 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7652 {
7653 PVM pVM = pVCpu->CTX_SUFF(pVM);
7654 Assert(PDMVmmDevHeapIsEnabled(pVM));
7655 Assert(pVM->hm.s.vmx.pRealModeTSS);
7656
7657 /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7658 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK
7659 | CPUMCTX_EXTRN_TABLE_MASK
7660 | CPUMCTX_EXTRN_RIP
7661 | CPUMCTX_EXTRN_RSP
7662 | CPUMCTX_EXTRN_RFLAGS);
7663 AssertRCReturn(rc, rc);
7664
7665 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7666 size_t const cbIdtEntry = sizeof(X86IDTR16);
7667 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7668 {
7669 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7670 if (uVector == X86_XCPT_DF)
7671 return VINF_EM_RESET;
7672
7673 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7674 if (uVector == X86_XCPT_GP)
7675 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, pfIntrState);
7676
7677 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7678 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7679 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
7680 fStepping, pfIntrState);
7681 }
7682
7683 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7684 uint16_t uGuestIp = pMixedCtx->ip;
7685 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7686 {
7687 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7688 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7689 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7690 }
7691 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7692 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7693
7694 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7695 X86IDTR16 IdtEntry;
7696 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7697 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7698 AssertRCReturn(rc, rc);
7699
7700 /* Construct the stack frame for the interrupt/exception handler. */
7701 VBOXSTRICTRC rcStrict;
7702 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7703 if (rcStrict == VINF_SUCCESS)
7704 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7705 if (rcStrict == VINF_SUCCESS)
7706 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7707
7708 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7709 if (rcStrict == VINF_SUCCESS)
7710 {
7711 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7712 pMixedCtx->rip = IdtEntry.offSel;
7713 pMixedCtx->cs.Sel = IdtEntry.uSel;
7714 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7715 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7716 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7717 && uVector == X86_XCPT_PF)
7718 pMixedCtx->cr2 = GCPtrFaultAddress;
7719
7720 /* If any other guest-state bits are changed here, make sure to update
7721 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7722 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS
7723 | HM_CHANGED_GUEST_CR2
7724 | HM_CHANGED_GUEST_RIP
7725 | HM_CHANGED_GUEST_RFLAGS
7726 | HM_CHANGED_GUEST_RSP);
7727
7728 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7729 if (*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7730 {
7731 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7732 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7733 Log4Func(("Clearing inhibition due to STI\n"));
7734 *pfIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7735 }
7736 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7737 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7738
7739 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7740 it, if we are returning to ring-3 before executing guest code. */
7741 pVCpu->hm.s.Event.fPending = false;
7742
7743 /* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
7744 if (fStepping)
7745 rcStrict = VINF_EM_DBG_STEPPED;
7746 }
7747 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7748 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7749 return rcStrict;
7750 }
7751 }
7752
7753 /* Validate. */
7754 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7755 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7756 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7757
7758 /* Inject. */
7759 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7760 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7761 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7762 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7763 AssertRCReturn(rc, rc);
7764
7765 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7766 && uVector == X86_XCPT_PF)
7767 pMixedCtx->cr2 = GCPtrFaultAddress;
7768
7769 Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr,
7770 pMixedCtx->cr2));
7771
7772 return VINF_SUCCESS;
7773}
7774
7775
7776/**
7777 * Clears the interrupt-window exiting control in the VMCS and if necessary
7778 * clears the current event in the VMCS as well.
7779 *
7780 * @returns VBox status code.
7781 * @param pVCpu The cross context virtual CPU structure.
7782 *
7783 * @remarks Use this function only to clear events that have not yet been
7784 * delivered to the guest but are injected in the VMCS!
7785 * @remarks No-long-jump zone!!!
7786 */
7787static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
7788{
7789 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7790 {
7791 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7792 Log4Func(("Cleared interrupt widow\n"));
7793 }
7794
7795 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
7796 {
7797 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
7798 Log4Func(("Cleared interrupt widow\n"));
7799 }
7800}
7801
7802
7803/**
7804 * Enters the VT-x session.
7805 *
7806 * @returns VBox status code.
7807 * @param pVM The cross context VM structure.
7808 * @param pVCpu The cross context virtual CPU structure.
7809 * @param pCpu Pointer to the CPU info struct.
7810 */
7811VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7812{
7813 AssertPtr(pVM);
7814 AssertPtr(pVCpu);
7815 Assert(pVM->hm.s.vmx.fSupported);
7816 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7817 NOREF(pCpu); NOREF(pVM);
7818
7819 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7820 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7821 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7822
7823#ifdef VBOX_STRICT
7824 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
7825 RTCCUINTREG uHostCR4 = ASMGetCR4();
7826 if (!(uHostCR4 & X86_CR4_VMXE))
7827 {
7828 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7829 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7830 }
7831#endif
7832
7833 /*
7834 * Load the VCPU's VMCS as the current (and active) one.
7835 */
7836 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7837 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7838 if (RT_FAILURE(rc))
7839 return rc;
7840
7841 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7842 pVCpu->hm.s.fLeaveDone = false;
7843 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7844
7845 return VINF_SUCCESS;
7846}
7847
7848
7849/**
7850 * The thread-context callback (only on platforms which support it).
7851 *
7852 * @param enmEvent The thread-context event.
7853 * @param pVCpu The cross context virtual CPU structure.
7854 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7855 * @thread EMT(pVCpu)
7856 */
7857VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7858{
7859 NOREF(fGlobalInit);
7860
7861 switch (enmEvent)
7862 {
7863 case RTTHREADCTXEVENT_OUT:
7864 {
7865 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7866 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7867 VMCPU_ASSERT_EMT(pVCpu);
7868
7869 /* No longjmps (logger flushes, locks) in this fragile context. */
7870 VMMRZCallRing3Disable(pVCpu);
7871 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7872
7873 /*
7874 * Restore host-state (FPU, debug etc.)
7875 */
7876 if (!pVCpu->hm.s.fLeaveDone)
7877 {
7878 /*
7879 * Do -not- import the guest-state here as we might already be in the middle of importing
7880 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
7881 */
7882 hmR0VmxLeave(pVCpu, false /* fImportState */);
7883 pVCpu->hm.s.fLeaveDone = true;
7884 }
7885
7886 /* Leave HM context, takes care of local init (term). */
7887 int rc = HMR0LeaveCpu(pVCpu);
7888 AssertRC(rc); NOREF(rc);
7889
7890 /* Restore longjmp state. */
7891 VMMRZCallRing3Enable(pVCpu);
7892 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
7893 break;
7894 }
7895
7896 case RTTHREADCTXEVENT_IN:
7897 {
7898 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7899 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7900 VMCPU_ASSERT_EMT(pVCpu);
7901
7902 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7903 VMMRZCallRing3Disable(pVCpu);
7904 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7905
7906 /* Initialize the bare minimum state required for HM. This takes care of
7907 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7908 int rc = HMR0EnterCpu(pVCpu);
7909 AssertRC(rc);
7910 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7911 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7912
7913 /* Load the active VMCS as the current one. */
7914 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7915 {
7916 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7917 AssertRC(rc); NOREF(rc);
7918 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7919 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7920 }
7921 pVCpu->hm.s.fLeaveDone = false;
7922
7923 /* Restore longjmp state. */
7924 VMMRZCallRing3Enable(pVCpu);
7925 break;
7926 }
7927
7928 default:
7929 break;
7930 }
7931}
7932
7933
7934/**
7935 * Exports the host state into the VMCS host-state area.
7936 * Sets up the VM-exit MSR-load area.
7937 *
7938 * The CPU state will be loaded from these fields on every successful VM-exit.
7939 *
7940 * @returns VBox status code.
7941 * @param pVCpu The cross context virtual CPU structure.
7942 *
7943 * @remarks No-long-jump zone!!!
7944 */
7945static int hmR0VmxExportHostState(PVMCPU pVCpu)
7946{
7947 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7948
7949 int rc = VINF_SUCCESS;
7950 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
7951 {
7952 rc = hmR0VmxExportHostControlRegs();
7953 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7954
7955 rc = hmR0VmxExportHostSegmentRegs(pVCpu);
7956 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7957
7958 rc = hmR0VmxExportHostMsrs(pVCpu);
7959 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7960
7961 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
7962 }
7963 return rc;
7964}
7965
7966
7967/**
7968 * Saves the host state in the VMCS host-state.
7969 *
7970 * @returns VBox status code.
7971 * @param pVM The cross context VM structure.
7972 * @param pVCpu The cross context virtual CPU structure.
7973 *
7974 * @remarks No-long-jump zone!!!
7975 */
7976VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
7977{
7978 AssertPtr(pVCpu);
7979 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7980
7981 /*
7982 * Export the host state here while entering HM context.
7983 * When thread-context hooks are used, we might get preempted and have to re-save the host
7984 * state but most of the time we won't be, so do it here before we disable interrupts.
7985 */
7986 return hmR0VmxExportHostState(pVCpu);
7987}
7988
7989
7990/**
7991 * Exports the guest state into the VMCS guest-state area.
7992 *
7993 * The will typically be done before VM-entry when the guest-CPU state and the
7994 * VMCS state may potentially be out of sync.
7995 *
7996 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
7997 * VM-entry controls.
7998 * Sets up the appropriate VMX non-root function to execute guest code based on
7999 * the guest CPU mode.
8000 *
8001 * @returns VBox strict status code.
8002 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8003 * without unrestricted guest access and the VMMDev is not presently
8004 * mapped (e.g. EFI32).
8005 *
8006 * @param pVM The cross context VM structure.
8007 * @param pVCpu The cross context virtual CPU structure.
8008 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8009 * out-of-sync. Make sure to update the required fields
8010 * before using them.
8011 *
8012 * @remarks No-long-jump zone!!!
8013 */
8014static VBOXSTRICTRC hmR0VmxExportGuestState(PVM pVM, PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
8015{
8016 AssertPtr(pVM);
8017 AssertPtr(pVCpu);
8018 AssertPtr(pMixedCtx);
8019 HMVMX_ASSERT_PREEMPT_SAFE();
8020
8021 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8022
8023 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
8024
8025 /* Determine real-on-v86 mode. */
8026 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8027 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8028 && CPUMIsGuestInRealModeEx(pMixedCtx))
8029 {
8030 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8031 }
8032
8033 /*
8034 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8035 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8036 */
8037 int rc = hmR0VmxSelectVMRunHandler(pVCpu, pMixedCtx);
8038 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8039
8040 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8041 rc = hmR0VmxExportGuestEntryCtls(pVCpu, pMixedCtx);
8042 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8043
8044 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8045 rc = hmR0VmxExportGuestExitCtls(pVCpu, pMixedCtx);
8046 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8047
8048 rc = hmR0VmxExportGuestCR0(pVCpu, pMixedCtx);
8049 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8050
8051 VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pMixedCtx);
8052 if (rcStrict == VINF_SUCCESS)
8053 { /* likely */ }
8054 else
8055 {
8056 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8057 return rcStrict;
8058 }
8059
8060 rc = hmR0VmxExportGuestSegmentRegs(pVCpu, pMixedCtx);
8061 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8062
8063 /* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8064 may alter controls if we determine we don't have to swap EFER after all. */
8065 rc = hmR0VmxExportGuestMsrs(pVCpu, pMixedCtx);
8066 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8067
8068 rc = hmR0VmxExportGuestApicTpr(pVCpu);
8069 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8070
8071 /* This needs to be done after hmR0VmxExportGuestCR0() as it may alter intercepted exceptions. */
8072 rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8073 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8074
8075 /* Exporting RFLAGS here is fine, even though RFLAGS.TF might depend on guest debug state which is
8076 not exported here. It is re-evaluated and updated if necessary in hmR0VmxExportSharedState(). */
8077 rc = hmR0VmxExportGuestRip(pVCpu, pMixedCtx);
8078 rc |= hmR0VmxExportGuestRsp(pVCpu, pMixedCtx);
8079 rc |= hmR0VmxExportGuestRflags(pVCpu, pMixedCtx);
8080 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8081
8082 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8083 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8084 | HM_CHANGED_GUEST_CR2
8085 | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8086 | HM_CHANGED_GUEST_X87
8087 | HM_CHANGED_GUEST_SSE_AVX
8088 | HM_CHANGED_GUEST_OTHER_XSAVE
8089 | HM_CHANGED_GUEST_XCRx
8090 | HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8091 | HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8092 | HM_CHANGED_GUEST_TSC_AUX
8093 | HM_CHANGED_GUEST_OTHER_MSRS
8094 | HM_CHANGED_GUEST_HWVIRT
8095 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8096
8097 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8098 return rc;
8099}
8100
8101
8102/**
8103 * Exports the state shared between the host and guest into the VMCS.
8104 *
8105 * @param pVM The cross context VM structure.
8106 * @param pVCpu The cross context virtual CPU structure.
8107 * @param pCtx Pointer to the guest-CPU context.
8108 *
8109 * @remarks No-long-jump zone!!!
8110 */
8111static void hmR0VmxExportSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8112{
8113 NOREF(pVM);
8114
8115 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8116 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8117
8118 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8119 {
8120 int rc = hmR0VmxExportSharedDebugState(pVCpu, pCtx);
8121 AssertRC(rc);
8122 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8123
8124 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8125 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8126 {
8127 rc = hmR0VmxExportGuestRflags(pVCpu, pCtx);
8128 AssertRC(rc);
8129 }
8130 }
8131
8132 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8133 {
8134 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8135 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8136 }
8137
8138 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8139 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8140}
8141
8142
8143/**
8144 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8145 *
8146 * @returns Strict VBox status code (i.e. informational status codes too).
8147 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8148 * without unrestricted guest access and the VMMDev is not presently
8149 * mapped (e.g. EFI32).
8150 *
8151 * @param pVM The cross context VM structure.
8152 * @param pVCpu The cross context virtual CPU structure.
8153 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8154 * out-of-sync. Make sure to update the required fields
8155 * before using them.
8156 *
8157 * @remarks No-long-jump zone!!!
8158 */
8159static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
8160{
8161 HMVMX_ASSERT_PREEMPT_SAFE();
8162 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8163 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8164
8165#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8166 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_ALL_GUEST;
8167#endif
8168
8169 /*
8170 * For many exits it's only RIP that changes and hence try to export it first
8171 * without going through a lot of change flag checks.
8172 */
8173 VBOXSTRICTRC rcStrict;
8174 uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8175 if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8176 {
8177 rcStrict = hmR0VmxExportGuestRip(pVCpu, pMixedCtx);
8178 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8179 { /* likely */}
8180 else
8181 AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8182 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8183 }
8184 else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8185 {
8186 rcStrict = hmR0VmxExportGuestState(pVM, pVCpu, pMixedCtx);
8187 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8188 { /* likely */}
8189 else
8190 {
8191 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8192 VBOXSTRICTRC_VAL(rcStrict)));
8193 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8194 return rcStrict;
8195 }
8196 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8197 }
8198 else
8199 rcStrict = VINF_SUCCESS;
8200
8201#ifdef VBOX_STRICT
8202 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8203 fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8204 AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8205 ("fCtxChanged=%#RX64\n", fCtxChanged));
8206#endif
8207 return rcStrict;
8208}
8209
8210
8211/**
8212 * Does the preparations before executing guest code in VT-x.
8213 *
8214 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8215 * recompiler/IEM. We must be cautious what we do here regarding committing
8216 * guest-state information into the VMCS assuming we assuredly execute the
8217 * guest in VT-x mode.
8218 *
8219 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8220 * the common-state (TRPM/forceflags), we must undo those changes so that the
8221 * recompiler/IEM can (and should) use them when it resumes guest execution.
8222 * Otherwise such operations must be done when we can no longer exit to ring-3.
8223 *
8224 * @returns Strict VBox status code (i.e. informational status codes too).
8225 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8226 * have been disabled.
8227 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8228 * double-fault into the guest.
8229 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8230 * dispatched directly.
8231 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8232 *
8233 * @param pVM The cross context VM structure.
8234 * @param pVCpu The cross context virtual CPU structure.
8235 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8236 * out-of-sync. Make sure to update the required fields
8237 * before using them.
8238 * @param pVmxTransient Pointer to the VMX transient structure.
8239 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8240 * us ignore some of the reasons for returning to
8241 * ring-3, and return VINF_EM_DBG_STEPPED if event
8242 * dispatching took place.
8243 */
8244static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8245{
8246 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8247
8248#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8249 PGMRZDynMapFlushAutoSet(pVCpu);
8250#endif
8251
8252 /* Check force flag actions that might require us to go back to ring-3. */
8253 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8254 if (rcStrict == VINF_SUCCESS)
8255 { /* FFs doesn't get set all the time. */ }
8256 else
8257 return rcStrict;
8258
8259 /*
8260 * Setup the virtualized-APIC accesses.
8261 *
8262 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8263 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8264 *
8265 * This is the reason we do it here and not in hmR0VmxExportGuestState().
8266 */
8267 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8268 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8269 && PDMHasApic(pVM))
8270 {
8271 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8272 Assert(u64MsrApicBase);
8273 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8274
8275 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8276
8277 /* Unalias any existing mapping. */
8278 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8279 AssertRCReturn(rc, rc);
8280
8281 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8282 Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8283 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8284 AssertRCReturn(rc, rc);
8285
8286 /* Update the per-VCPU cache of the APIC base MSR. */
8287 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8288 }
8289
8290 if (TRPMHasTrap(pVCpu))
8291 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8292 uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8293
8294 /*
8295 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8296 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8297 */
8298 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, fIntrState, fStepping);
8299 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8300 { /* likely */ }
8301 else
8302 {
8303 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8304 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8305 return rcStrict;
8306 }
8307
8308 /*
8309 * No longjmps to ring-3 from this point on!!!
8310 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8311 * This also disables flushing of the R0-logger instance (if any).
8312 */
8313 VMMRZCallRing3Disable(pVCpu);
8314
8315 /*
8316 * Export the guest state bits.
8317 *
8318 * We cannot perform longjmps while loading the guest state because we do not preserve the
8319 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8320 * CPU migration.
8321 *
8322 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8323 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8324 * Hence, loading of the guest state needs to be done -after- injection of events.
8325 */
8326 rcStrict = hmR0VmxExportGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8327 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8328 { /* likely */ }
8329 else
8330 {
8331 VMMRZCallRing3Enable(pVCpu);
8332 return rcStrict;
8333 }
8334
8335 /*
8336 * We disable interrupts so that we don't miss any interrupts that would flag
8337 * preemption (IPI/timers etc.) when thread-context hooks aren't used and we've
8338 * been running with preemption disabled for a while. Since this is purly to aid
8339 * the RTThreadPreemptIsPending code, it doesn't matter that it may temporarily
8340 * reenable and disable interrupt on NT.
8341 *
8342 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8343 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8344 *
8345 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8346 * executing guest code.
8347 */
8348 pVmxTransient->fEFlags = ASMIntDisableFlags();
8349
8350 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8351 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8352 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8353 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8354 {
8355 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8356 {
8357 pVCpu->hm.s.Event.fPending = false;
8358
8359 /*
8360 * We've injected any pending events. This is really the point of no return (to ring-3).
8361 *
8362 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8363 * returns from this function, so don't enable them here.
8364 */
8365 return VINF_SUCCESS;
8366 }
8367
8368 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8369 rcStrict = VINF_EM_RAW_INTERRUPT;
8370 }
8371 else
8372 {
8373 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8374 rcStrict = VINF_EM_RAW_TO_R3;
8375 }
8376
8377 ASMSetFlags(pVmxTransient->fEFlags);
8378 VMMRZCallRing3Enable(pVCpu);
8379
8380 return rcStrict;
8381}
8382
8383
8384/**
8385 * Prepares to run guest code in VT-x and we've committed to doing so. This
8386 * means there is no backing out to ring-3 or anywhere else at this
8387 * point.
8388 *
8389 * @param pVM The cross context VM structure.
8390 * @param pVCpu The cross context virtual CPU structure.
8391 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8392 * out-of-sync. Make sure to update the required fields
8393 * before using them.
8394 * @param pVmxTransient Pointer to the VMX transient structure.
8395 *
8396 * @remarks Called with preemption disabled.
8397 * @remarks No-long-jump zone!!!
8398 */
8399static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8400{
8401 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8402 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8403 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8404
8405 /*
8406 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8407 */
8408 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8409 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8410
8411 if (!CPUMIsGuestFPUStateActive(pVCpu))
8412 {
8413 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8414 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8415 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
8416 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8417 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8418 }
8419
8420 /*
8421 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8422 */
8423 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8424 && pVCpu->hm.s.vmx.cMsrs > 0)
8425 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8426
8427 /*
8428 * Re-save the host state bits as we may've been preempted (only happens when
8429 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8430 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8431 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8432 * See @bugref{8432}.
8433 */
8434 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8435 {
8436 int rc = hmR0VmxExportHostState(pVCpu);
8437 AssertRC(rc);
8438 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8439 }
8440 Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8441
8442 /*
8443 * Export the state shared between host and guest (FPU, debug, lazy MSRs).
8444 */
8445 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8446 hmR0VmxExportSharedState(pVM, pVCpu, pMixedCtx);
8447 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8448
8449 /* Store status of the shared guest-host state at the time of VM-entry. */
8450#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8451 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8452 {
8453 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8454 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8455 }
8456 else
8457#endif
8458 {
8459 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8460 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8461 }
8462
8463 /*
8464 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8465 */
8466 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8467 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8468
8469 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8470 RTCPUID idCurrentCpu = pCpu->idCpu;
8471 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8472 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8473 {
8474 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8475 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8476 }
8477
8478 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8479 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8480 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8481 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8482
8483 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8484
8485 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8486 to start executing. */
8487
8488 /*
8489 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8490 */
8491 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8492 {
8493 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8494 {
8495 bool fMsrUpdated;
8496 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8497 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8498 &fMsrUpdated);
8499 AssertRC(rc2);
8500 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8501 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8502 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8503 }
8504 else
8505 {
8506 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8507 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8508 }
8509 }
8510
8511 if (pVM->cpum.ro.GuestFeatures.fIbrs)
8512 {
8513 bool fMsrUpdated;
8514 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8515 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8516 &fMsrUpdated);
8517 AssertRC(rc2);
8518 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8519 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8520 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8521 }
8522
8523#ifdef VBOX_STRICT
8524 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8525 hmR0VmxCheckHostEferMsr(pVCpu);
8526 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8527#endif
8528#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8529 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8530 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8531 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8532#endif
8533}
8534
8535
8536/**
8537 * Performs some essential restoration of state after running guest code in
8538 * VT-x.
8539 *
8540 * @param pVM The cross context VM structure.
8541 * @param pVCpu The cross context virtual CPU structure.
8542 * @param pVmxTransient Pointer to the VMX transient structure.
8543 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8544 *
8545 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8546 *
8547 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8548 * unconditionally when it is safe to do so.
8549 */
8550static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8551{
8552 uint64_t const uHostTsc = ASMReadTSC();
8553 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8554
8555 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8556 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8557 pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8558 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8559 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8560 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8561
8562 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8563 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TSCOffset);
8564
8565 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8566 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8567 Assert(!ASMIntAreEnabled());
8568 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8569
8570#if HC_ARCH_BITS == 64
8571 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8572#endif
8573#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8574 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8575 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8576 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8577#else
8578 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8579#endif
8580#ifdef VBOX_STRICT
8581 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8582#endif
8583 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8584
8585 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8586 uint32_t uExitReason;
8587 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8588 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8589 AssertRC(rc);
8590 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8591 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8592
8593 if (rcVMRun == VINF_SUCCESS)
8594 {
8595 /*
8596 * Update the VM-exit history array here even if the VM-entry failed due to:
8597 * - Invalid guest state.
8598 * - MSR loading.
8599 * - Machine-check event.
8600 *
8601 * In any of the above cases we will still have a "valid" VM-exit reason
8602 * despite @a fVMEntryFailed being false.
8603 *
8604 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8605 *
8606 * Note! We don't have CS or RIP at this point. Will probably address that later
8607 * by amending the history entry added here.
8608 */
8609 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8610 UINT64_MAX, uHostTsc);
8611
8612 if (!pVmxTransient->fVMEntryFailed)
8613 {
8614 VMMRZCallRing3Enable(pVCpu);
8615
8616 /*
8617 * Import the guest-interruptibility state always as we need it while evaluating
8618 * injecting events on re-entry.
8619 *
8620 * We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8621 * checking for real-mode while exporting the state because all bits that cause
8622 * mode changes wrt CR0 are intercepted.
8623 */
8624 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8625 AssertRC(rc);
8626
8627#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8628 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8629 AssertRC(rc);
8630#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8631 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8632 AssertRC(rc);
8633#endif
8634
8635 /*
8636 * Sync the TPR shadow with our APIC state.
8637 */
8638 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8639 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8640 {
8641 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8642 AssertRC(rc);
8643 ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8644 }
8645
8646 return;
8647 }
8648 }
8649 else
8650 {
8651 Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8652 }
8653
8654 VMMRZCallRing3Enable(pVCpu);
8655}
8656
8657
8658/**
8659 * Runs the guest code using VT-x the normal way.
8660 *
8661 * @returns VBox status code.
8662 * @param pVM The cross context VM structure.
8663 * @param pVCpu The cross context virtual CPU structure.
8664 * @param pCtx Pointer to the guest-CPU context.
8665 *
8666 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8667 */
8668static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8669{
8670 VMXTRANSIENT VmxTransient;
8671 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8672 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8673 uint32_t cLoops = 0;
8674
8675 for (;; cLoops++)
8676 {
8677 Assert(!HMR0SuspendPending());
8678 HMVMX_ASSERT_CPU_SAFE();
8679
8680 /* Preparatory work for running guest code, this may force us to return
8681 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8682 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8683 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8684 if (rcStrict != VINF_SUCCESS)
8685 break;
8686
8687 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8688 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8689 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8690
8691 /* Restore any residual host-state and save any bits shared between host
8692 and guest into the guest-CPU state. Re-enables interrupts! */
8693 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8694
8695 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8696 if (RT_SUCCESS(rcRun))
8697 { /* very likely */ }
8698 else
8699 {
8700 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8701 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
8702 return rcRun;
8703 }
8704
8705 /* Profile the VM-exit. */
8706 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8707 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8708 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8709 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8710 HMVMX_START_EXIT_DISPATCH_PROF();
8711
8712 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8713
8714 /* Handle the VM-exit. */
8715#ifdef HMVMX_USE_FUNCTION_TABLE
8716 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8717#else
8718 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8719#endif
8720 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8721 if (rcStrict == VINF_SUCCESS)
8722 {
8723 if (cLoops <= pVM->hm.s.cMaxResumeLoops)
8724 continue; /* likely */
8725 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8726 rcStrict = VINF_EM_RAW_INTERRUPT;
8727 }
8728 break;
8729 }
8730
8731 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8732 return rcStrict;
8733}
8734
8735
8736
8737/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8738 * probes.
8739 *
8740 * The following few functions and associated structure contains the bloat
8741 * necessary for providing detailed debug events and dtrace probes as well as
8742 * reliable host side single stepping. This works on the principle of
8743 * "subclassing" the normal execution loop and workers. We replace the loop
8744 * method completely and override selected helpers to add necessary adjustments
8745 * to their core operation.
8746 *
8747 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8748 * any performance for debug and analysis features.
8749 *
8750 * @{
8751 */
8752
8753/**
8754 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
8755 * the debug run loop.
8756 */
8757typedef struct VMXRUNDBGSTATE
8758{
8759 /** The RIP we started executing at. This is for detecting that we stepped. */
8760 uint64_t uRipStart;
8761 /** The CS we started executing with. */
8762 uint16_t uCsStart;
8763
8764 /** Whether we've actually modified the 1st execution control field. */
8765 bool fModifiedProcCtls : 1;
8766 /** Whether we've actually modified the 2nd execution control field. */
8767 bool fModifiedProcCtls2 : 1;
8768 /** Whether we've actually modified the exception bitmap. */
8769 bool fModifiedXcptBitmap : 1;
8770
8771 /** We desire the modified the CR0 mask to be cleared. */
8772 bool fClearCr0Mask : 1;
8773 /** We desire the modified the CR4 mask to be cleared. */
8774 bool fClearCr4Mask : 1;
8775 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8776 uint32_t fCpe1Extra;
8777 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
8778 uint32_t fCpe1Unwanted;
8779 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
8780 uint32_t fCpe2Extra;
8781 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
8782 uint32_t bmXcptExtra;
8783 /** The sequence number of the Dtrace provider settings the state was
8784 * configured against. */
8785 uint32_t uDtraceSettingsSeqNo;
8786 /** VM-exits to check (one bit per VM-exit). */
8787 uint32_t bmExitsToCheck[3];
8788
8789 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
8790 uint32_t fProcCtlsInitial;
8791 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
8792 uint32_t fProcCtls2Initial;
8793 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
8794 uint32_t bmXcptInitial;
8795} VMXRUNDBGSTATE;
8796AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
8797typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
8798
8799
8800/**
8801 * Initializes the VMXRUNDBGSTATE structure.
8802 *
8803 * @param pVCpu The cross context virtual CPU structure of the
8804 * calling EMT.
8805 * @param pCtx The CPU register context to go with @a pVCpu.
8806 * @param pDbgState The structure to initialize.
8807 */
8808static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
8809{
8810 pDbgState->uRipStart = pCtx->rip;
8811 pDbgState->uCsStart = pCtx->cs.Sel;
8812
8813 pDbgState->fModifiedProcCtls = false;
8814 pDbgState->fModifiedProcCtls2 = false;
8815 pDbgState->fModifiedXcptBitmap = false;
8816 pDbgState->fClearCr0Mask = false;
8817 pDbgState->fClearCr4Mask = false;
8818 pDbgState->fCpe1Extra = 0;
8819 pDbgState->fCpe1Unwanted = 0;
8820 pDbgState->fCpe2Extra = 0;
8821 pDbgState->bmXcptExtra = 0;
8822 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
8823 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
8824 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
8825}
8826
8827
8828/**
8829 * Updates the VMSC fields with changes requested by @a pDbgState.
8830 *
8831 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
8832 * immediately before executing guest code, i.e. when interrupts are disabled.
8833 * We don't check status codes here as we cannot easily assert or return in the
8834 * latter case.
8835 *
8836 * @param pVCpu The cross context virtual CPU structure.
8837 * @param pDbgState The debug state.
8838 */
8839static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8840{
8841 /*
8842 * Ensure desired flags in VMCS control fields are set.
8843 * (Ignoring write failure here, as we're committed and it's just debug extras.)
8844 *
8845 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
8846 * there should be no stale data in pCtx at this point.
8847 */
8848 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
8849 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
8850 {
8851 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
8852 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
8853 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8854 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
8855 pDbgState->fModifiedProcCtls = true;
8856 }
8857
8858 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
8859 {
8860 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
8861 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
8862 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
8863 pDbgState->fModifiedProcCtls2 = true;
8864 }
8865
8866 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
8867 {
8868 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
8869 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8870 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
8871 pDbgState->fModifiedXcptBitmap = true;
8872 }
8873
8874 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
8875 {
8876 pVCpu->hm.s.vmx.u32CR0Mask = 0;
8877 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
8878 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
8879 }
8880
8881 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
8882 {
8883 pVCpu->hm.s.vmx.u32CR4Mask = 0;
8884 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
8885 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
8886 }
8887}
8888
8889
8890static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
8891{
8892 /*
8893 * Restore VM-exit control settings as we may not reenter this function the
8894 * next time around.
8895 */
8896 /* We reload the initial value, trigger what we can of recalculations the
8897 next time around. From the looks of things, that's all that's required atm. */
8898 if (pDbgState->fModifiedProcCtls)
8899 {
8900 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
8901 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
8902 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
8903 AssertRCReturn(rc2, rc2);
8904 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
8905 }
8906
8907 /* We're currently the only ones messing with this one, so just restore the
8908 cached value and reload the field. */
8909 if ( pDbgState->fModifiedProcCtls2
8910 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
8911 {
8912 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
8913 AssertRCReturn(rc2, rc2);
8914 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
8915 }
8916
8917 /* If we've modified the exception bitmap, we restore it and trigger
8918 reloading and partial recalculation the next time around. */
8919 if (pDbgState->fModifiedXcptBitmap)
8920 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
8921
8922 return rcStrict;
8923}
8924
8925
8926/**
8927 * Configures VM-exit controls for current DBGF and DTrace settings.
8928 *
8929 * This updates @a pDbgState and the VMCS execution control fields to reflect
8930 * the necessary VM-exits demanded by DBGF and DTrace.
8931 *
8932 * @param pVCpu The cross context virtual CPU structure.
8933 * @param pDbgState The debug state.
8934 * @param pVmxTransient Pointer to the VMX transient structure. May update
8935 * fUpdateTscOffsettingAndPreemptTimer.
8936 */
8937static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
8938{
8939 /*
8940 * Take down the dtrace serial number so we can spot changes.
8941 */
8942 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
8943 ASMCompilerBarrier();
8944
8945 /*
8946 * We'll rebuild most of the middle block of data members (holding the
8947 * current settings) as we go along here, so start by clearing it all.
8948 */
8949 pDbgState->bmXcptExtra = 0;
8950 pDbgState->fCpe1Extra = 0;
8951 pDbgState->fCpe1Unwanted = 0;
8952 pDbgState->fCpe2Extra = 0;
8953 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
8954 pDbgState->bmExitsToCheck[i] = 0;
8955
8956 /*
8957 * Software interrupts (INT XXh) - no idea how to trigger these...
8958 */
8959 PVM pVM = pVCpu->CTX_SUFF(pVM);
8960 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
8961 || VBOXVMM_INT_SOFTWARE_ENABLED())
8962 {
8963 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
8964 }
8965
8966 /*
8967 * INT3 breakpoints - triggered by #BP exceptions.
8968 */
8969 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
8970 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
8971
8972 /*
8973 * Exception bitmap and XCPT events+probes.
8974 */
8975 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
8976 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
8977 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
8978
8979 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
8980 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
8981 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
8982 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
8983 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
8984 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
8985 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
8986 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
8987 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
8988 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
8989 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
8990 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
8991 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
8992 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
8993 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
8994 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
8995 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
8996 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
8997
8998 if (pDbgState->bmXcptExtra)
8999 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9000
9001 /*
9002 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9003 *
9004 * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
9005 * So, when adding/changing/removing please don't forget to update it.
9006 *
9007 * Some of the macros are picking up local variables to save horizontal space,
9008 * (being able to see it in a table is the lesser evil here).
9009 */
9010#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9011 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9012 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9013#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9014 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9015 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9016 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9017 } else do { } while (0)
9018#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9019 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9020 { \
9021 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9022 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9023 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9024 } else do { } while (0)
9025#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9026 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9027 { \
9028 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9029 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9030 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9031 } else do { } while (0)
9032#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9033 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9034 { \
9035 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9036 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9037 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9038 } else do { } while (0)
9039
9040 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9041 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9042 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9043 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9044 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9045
9046 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9047 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9048 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9049 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9050 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9051 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9052 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9053 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9054 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9055 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9056 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9057 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9058 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9059 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9060 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9061 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9062 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9063 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9064 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9065 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9066 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9067 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9068 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9069 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9070 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9071 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9072 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9073 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9074 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9075 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9076 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9077 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9078 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9079 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9080 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9081 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9082
9083 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9084 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9085 {
9086 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
9087 | CPUMCTX_EXTRN_CR4
9088 | CPUMCTX_EXTRN_APIC_TPR);
9089 AssertRC(rc);
9090
9091#if 0 /** @todo fix me */
9092 pDbgState->fClearCr0Mask = true;
9093 pDbgState->fClearCr4Mask = true;
9094#endif
9095 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9096 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9097 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9098 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9099 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9100 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9101 require clearing here and in the loop if we start using it. */
9102 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9103 }
9104 else
9105 {
9106 if (pDbgState->fClearCr0Mask)
9107 {
9108 pDbgState->fClearCr0Mask = false;
9109 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9110 }
9111 if (pDbgState->fClearCr4Mask)
9112 {
9113 pDbgState->fClearCr4Mask = false;
9114 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9115 }
9116 }
9117 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9118 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9119
9120 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9121 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9122 {
9123 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9124 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9125 }
9126 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9127 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9128
9129 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9130 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9131 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9132 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9133 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9134 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9135 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9136 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9137#if 0 /** @todo too slow, fix handler. */
9138 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9139#endif
9140 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9141
9142 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9143 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9144 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9145 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9146 {
9147 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9148 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9149 }
9150 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9151 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9152 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9153 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9154
9155 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9156 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9157 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9158 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9159 {
9160 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9161 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9162 }
9163 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9164 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9165 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9166 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9167
9168 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9169 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9170 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9171 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9172 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9173 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9174 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9175 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9176 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9177 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9178 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9179 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9180 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9181 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9182 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9183 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9184 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9185 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9186 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9187 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9188 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9189 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9190
9191#undef IS_EITHER_ENABLED
9192#undef SET_ONLY_XBM_IF_EITHER_EN
9193#undef SET_CPE1_XBM_IF_EITHER_EN
9194#undef SET_CPEU_XBM_IF_EITHER_EN
9195#undef SET_CPE2_XBM_IF_EITHER_EN
9196
9197 /*
9198 * Sanitize the control stuff.
9199 */
9200 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9201 if (pDbgState->fCpe2Extra)
9202 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9203 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9204 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9205 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9206 {
9207 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9208 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9209 }
9210
9211 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9212 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9213 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9214 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9215}
9216
9217
9218/**
9219 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9220 * appropriate.
9221 *
9222 * The caller has checked the VM-exit against the
9223 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9224 * already, so we don't have to do that either.
9225 *
9226 * @returns Strict VBox status code (i.e. informational status codes too).
9227 * @param pVM The cross context VM structure.
9228 * @param pVCpu The cross context virtual CPU structure.
9229 * @param pMixedCtx Pointer to the guest-CPU context.
9230 * @param pVmxTransient Pointer to the VMX-transient structure.
9231 * @param uExitReason The VM-exit reason.
9232 *
9233 * @remarks The name of this function is displayed by dtrace, so keep it short
9234 * and to the point. No longer than 33 chars long, please.
9235 */
9236static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9237 PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9238{
9239 /*
9240 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9241 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9242 *
9243 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9244 * does. Must add/change/remove both places. Same ordering, please.
9245 *
9246 * Added/removed events must also be reflected in the next section
9247 * where we dispatch dtrace events.
9248 */
9249 bool fDtrace1 = false;
9250 bool fDtrace2 = false;
9251 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9252 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9253 uint32_t uEventArg = 0;
9254#define SET_EXIT(a_EventSubName) \
9255 do { \
9256 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9257 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9258 } while (0)
9259#define SET_BOTH(a_EventSubName) \
9260 do { \
9261 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9262 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9263 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9264 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9265 } while (0)
9266 switch (uExitReason)
9267 {
9268 case VMX_EXIT_MTF:
9269 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9270
9271 case VMX_EXIT_XCPT_OR_NMI:
9272 {
9273 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9274 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9275 {
9276 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9277 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9278 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9279 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9280 {
9281 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9282 {
9283 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9284 uEventArg = pVmxTransient->uExitIntErrorCode;
9285 }
9286 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9287 switch (enmEvent1)
9288 {
9289 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9290 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9291 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9292 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9293 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9294 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9295 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9296 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9297 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9298 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9299 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9300 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9301 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9302 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9303 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9304 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9305 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9306 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9307 default: break;
9308 }
9309 }
9310 else
9311 AssertFailed();
9312 break;
9313
9314 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9315 uEventArg = idxVector;
9316 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9317 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9318 break;
9319 }
9320 break;
9321 }
9322
9323 case VMX_EXIT_TRIPLE_FAULT:
9324 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9325 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9326 break;
9327 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9328 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9329 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9330 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9331 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9332
9333 /* Instruction specific VM-exits: */
9334 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9335 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9336 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9337 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9338 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9339 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9340 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9341 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9342 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9343 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9344 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9345 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9346 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9347 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9348 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9349 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9350 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9351 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9352 case VMX_EXIT_MOV_CRX:
9353 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9354 if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9355 SET_BOTH(CRX_READ);
9356 else
9357 SET_BOTH(CRX_WRITE);
9358 uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQualification);
9359 break;
9360 case VMX_EXIT_MOV_DRX:
9361 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9362 if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification)
9363 == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9364 SET_BOTH(DRX_READ);
9365 else
9366 SET_BOTH(DRX_WRITE);
9367 uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification);
9368 break;
9369 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9370 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9371 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9372 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9373 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9374 case VMX_EXIT_XDTR_ACCESS:
9375 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9376 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9377 {
9378 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9379 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9380 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9381 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9382 }
9383 break;
9384
9385 case VMX_EXIT_TR_ACCESS:
9386 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9387 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9388 {
9389 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9390 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9391 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9392 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9393 }
9394 break;
9395
9396 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9397 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9398 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9399 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9400 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9401 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9402 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9403 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9404 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9405 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9406 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9407
9408 /* Events that aren't relevant at this point. */
9409 case VMX_EXIT_EXT_INT:
9410 case VMX_EXIT_INT_WINDOW:
9411 case VMX_EXIT_NMI_WINDOW:
9412 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9413 case VMX_EXIT_PREEMPT_TIMER:
9414 case VMX_EXIT_IO_INSTR:
9415 break;
9416
9417 /* Errors and unexpected events. */
9418 case VMX_EXIT_INIT_SIGNAL:
9419 case VMX_EXIT_SIPI:
9420 case VMX_EXIT_IO_SMI:
9421 case VMX_EXIT_SMI:
9422 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9423 case VMX_EXIT_ERR_MSR_LOAD:
9424 case VMX_EXIT_ERR_MACHINE_CHECK:
9425 break;
9426
9427 default:
9428 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9429 break;
9430 }
9431#undef SET_BOTH
9432#undef SET_EXIT
9433
9434 /*
9435 * Dtrace tracepoints go first. We do them here at once so we don't
9436 * have to copy the guest state saving and stuff a few dozen times.
9437 * Down side is that we've got to repeat the switch, though this time
9438 * we use enmEvent since the probes are a subset of what DBGF does.
9439 */
9440 if (fDtrace1 || fDtrace2)
9441 {
9442 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9443 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9444 switch (enmEvent1)
9445 {
9446 /** @todo consider which extra parameters would be helpful for each probe. */
9447 case DBGFEVENT_END: break;
9448 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9449 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9450 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9451 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9452 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9453 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9454 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9455 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9456 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9457 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9458 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9459 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9460 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9461 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9462 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9463 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9464 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9465 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9466 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9467 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9468 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9469 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9470 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9471 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9472 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9473 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9474 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9475 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9476 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9477 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9478 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9479 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9480 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9481 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9482 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9483 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9484 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9485 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9486 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9487 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9488 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9489 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9490 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9491 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9492 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9493 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9494 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9495 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9496 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9497 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9498 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9499 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9500 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9501 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9502 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9503 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9504 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9505 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9506 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9507 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9508 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9509 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9510 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9511 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9512 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9513 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9514 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9515 }
9516 switch (enmEvent2)
9517 {
9518 /** @todo consider which extra parameters would be helpful for each probe. */
9519 case DBGFEVENT_END: break;
9520 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9521 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9522 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9523 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9524 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9525 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9526 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9527 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9528 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9529 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9530 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9531 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9532 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9533 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9534 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9535 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9536 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9537 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9538 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9539 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9540 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9541 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9542 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9543 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9544 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9545 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9546 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9547 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9548 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9549 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9550 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9551 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9552 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9553 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9554 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9555 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9556 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9557 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9558 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9559 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9560 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9561 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9562 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9563 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9564 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9565 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9566 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9567 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9568 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9569 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9570 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9571 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9572 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9573 }
9574 }
9575
9576 /*
9577 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9578 * the DBGF call will do a full check).
9579 *
9580 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9581 * Note! If we have to events, we prioritize the first, i.e. the instruction
9582 * one, in order to avoid event nesting.
9583 */
9584 if ( enmEvent1 != DBGFEVENT_END
9585 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9586 {
9587 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9588 if (rcStrict != VINF_SUCCESS)
9589 return rcStrict;
9590 }
9591 else if ( enmEvent2 != DBGFEVENT_END
9592 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9593 {
9594 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9595 if (rcStrict != VINF_SUCCESS)
9596 return rcStrict;
9597 }
9598
9599 return VINF_SUCCESS;
9600}
9601
9602
9603/**
9604 * Single-stepping VM-exit filtering.
9605 *
9606 * This is preprocessing the VM-exits and deciding whether we've gotten far
9607 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9608 * handling is performed.
9609 *
9610 * @returns Strict VBox status code (i.e. informational status codes too).
9611 * @param pVM The cross context VM structure.
9612 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9613 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9614 * out-of-sync. Make sure to update the required
9615 * fields before using them.
9616 * @param pVmxTransient Pointer to the VMX-transient structure.
9617 * @param uExitReason The VM-exit reason.
9618 * @param pDbgState The debug state.
9619 */
9620DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9621 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9622{
9623 /*
9624 * Expensive (saves context) generic dtrace VM-exit probe.
9625 */
9626 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9627 { /* more likely */ }
9628 else
9629 {
9630 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9631 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9632 AssertRC(rc);
9633 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9634 }
9635
9636 /*
9637 * Check for host NMI, just to get that out of the way.
9638 */
9639 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9640 { /* normally likely */ }
9641 else
9642 {
9643 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9644 AssertRCReturn(rc2, rc2);
9645 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9646 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9647 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9648 }
9649
9650 /*
9651 * Check for single stepping event if we're stepping.
9652 */
9653 if (pVCpu->hm.s.fSingleInstruction)
9654 {
9655 switch (uExitReason)
9656 {
9657 case VMX_EXIT_MTF:
9658 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9659
9660 /* Various events: */
9661 case VMX_EXIT_XCPT_OR_NMI:
9662 case VMX_EXIT_EXT_INT:
9663 case VMX_EXIT_TRIPLE_FAULT:
9664 case VMX_EXIT_INT_WINDOW:
9665 case VMX_EXIT_NMI_WINDOW:
9666 case VMX_EXIT_TASK_SWITCH:
9667 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9668 case VMX_EXIT_APIC_ACCESS:
9669 case VMX_EXIT_EPT_VIOLATION:
9670 case VMX_EXIT_EPT_MISCONFIG:
9671 case VMX_EXIT_PREEMPT_TIMER:
9672
9673 /* Instruction specific VM-exits: */
9674 case VMX_EXIT_CPUID:
9675 case VMX_EXIT_GETSEC:
9676 case VMX_EXIT_HLT:
9677 case VMX_EXIT_INVD:
9678 case VMX_EXIT_INVLPG:
9679 case VMX_EXIT_RDPMC:
9680 case VMX_EXIT_RDTSC:
9681 case VMX_EXIT_RSM:
9682 case VMX_EXIT_VMCALL:
9683 case VMX_EXIT_VMCLEAR:
9684 case VMX_EXIT_VMLAUNCH:
9685 case VMX_EXIT_VMPTRLD:
9686 case VMX_EXIT_VMPTRST:
9687 case VMX_EXIT_VMREAD:
9688 case VMX_EXIT_VMRESUME:
9689 case VMX_EXIT_VMWRITE:
9690 case VMX_EXIT_VMXOFF:
9691 case VMX_EXIT_VMXON:
9692 case VMX_EXIT_MOV_CRX:
9693 case VMX_EXIT_MOV_DRX:
9694 case VMX_EXIT_IO_INSTR:
9695 case VMX_EXIT_RDMSR:
9696 case VMX_EXIT_WRMSR:
9697 case VMX_EXIT_MWAIT:
9698 case VMX_EXIT_MONITOR:
9699 case VMX_EXIT_PAUSE:
9700 case VMX_EXIT_XDTR_ACCESS:
9701 case VMX_EXIT_TR_ACCESS:
9702 case VMX_EXIT_INVEPT:
9703 case VMX_EXIT_RDTSCP:
9704 case VMX_EXIT_INVVPID:
9705 case VMX_EXIT_WBINVD:
9706 case VMX_EXIT_XSETBV:
9707 case VMX_EXIT_RDRAND:
9708 case VMX_EXIT_INVPCID:
9709 case VMX_EXIT_VMFUNC:
9710 case VMX_EXIT_RDSEED:
9711 case VMX_EXIT_XSAVES:
9712 case VMX_EXIT_XRSTORS:
9713 {
9714 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
9715 | CPUMCTX_EXTRN_CS);
9716 AssertRCReturn(rc, rc);
9717 if ( pMixedCtx->rip != pDbgState->uRipStart
9718 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
9719 return VINF_EM_DBG_STEPPED;
9720 break;
9721 }
9722
9723 /* Errors and unexpected events: */
9724 case VMX_EXIT_INIT_SIGNAL:
9725 case VMX_EXIT_SIPI:
9726 case VMX_EXIT_IO_SMI:
9727 case VMX_EXIT_SMI:
9728 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9729 case VMX_EXIT_ERR_MSR_LOAD:
9730 case VMX_EXIT_ERR_MACHINE_CHECK:
9731 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9732 break;
9733
9734 default:
9735 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9736 break;
9737 }
9738 }
9739
9740 /*
9741 * Check for debugger event breakpoints and dtrace probes.
9742 */
9743 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9744 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9745 {
9746 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9747 if (rcStrict != VINF_SUCCESS)
9748 return rcStrict;
9749 }
9750
9751 /*
9752 * Normal processing.
9753 */
9754#ifdef HMVMX_USE_FUNCTION_TABLE
9755 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9756#else
9757 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9758#endif
9759}
9760
9761
9762/**
9763 * Single steps guest code using VT-x.
9764 *
9765 * @returns Strict VBox status code (i.e. informational status codes too).
9766 * @param pVM The cross context VM structure.
9767 * @param pVCpu The cross context virtual CPU structure.
9768 * @param pCtx Pointer to the guest-CPU context.
9769 *
9770 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
9771 */
9772static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9773{
9774 VMXTRANSIENT VmxTransient;
9775 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9776
9777 /* Set HMCPU indicators. */
9778 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
9779 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
9780 pVCpu->hm.s.fDebugWantRdTscExit = false;
9781 pVCpu->hm.s.fUsingDebugLoop = true;
9782
9783 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
9784 VMXRUNDBGSTATE DbgState;
9785 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
9786 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9787
9788 /*
9789 * The loop.
9790 */
9791 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9792 for (uint32_t cLoops = 0; ; cLoops++)
9793 {
9794 Assert(!HMR0SuspendPending());
9795 HMVMX_ASSERT_CPU_SAFE();
9796 bool fStepping = pVCpu->hm.s.fSingleInstruction;
9797
9798 /*
9799 * Preparatory work for running guest code, this may force us to return
9800 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
9801 */
9802 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9803 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
9804 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
9805 if (rcStrict != VINF_SUCCESS)
9806 break;
9807
9808 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9809 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
9810
9811 /*
9812 * Now we can run the guest code.
9813 */
9814 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9815
9816 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9817
9818 /*
9819 * Restore any residual host-state and save any bits shared between host
9820 * and guest into the guest-CPU state. Re-enables interrupts!
9821 */
9822 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
9823
9824 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9825 if (RT_SUCCESS(rcRun))
9826 { /* very likely */ }
9827 else
9828 {
9829 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9830 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9831 return rcRun;
9832 }
9833
9834 /* Profile the VM-exit. */
9835 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9836 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9837 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9838 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9839 HMVMX_START_EXIT_DISPATCH_PROF();
9840
9841 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9842
9843 /*
9844 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
9845 */
9846 rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
9847 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9848 if (rcStrict != VINF_SUCCESS)
9849 break;
9850 if (cLoops > pVM->hm.s.cMaxResumeLoops)
9851 {
9852 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9853 rcStrict = VINF_EM_RAW_INTERRUPT;
9854 break;
9855 }
9856
9857 /*
9858 * Stepping: Did the RIP change, if so, consider it a single step.
9859 * Otherwise, make sure one of the TFs gets set.
9860 */
9861 if (fStepping)
9862 {
9863 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
9864 | CPUMCTX_EXTRN_CS);
9865 AssertRC(rc);
9866 if ( pCtx->rip != DbgState.uRipStart
9867 || pCtx->cs.Sel != DbgState.uCsStart)
9868 {
9869 rcStrict = VINF_EM_DBG_STEPPED;
9870 break;
9871 }
9872 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
9873 }
9874
9875 /*
9876 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
9877 */
9878 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
9879 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9880 }
9881
9882 /*
9883 * Clear the X86_EFL_TF if necessary.
9884 */
9885 if (pVCpu->hm.s.fClearTrapFlag)
9886 {
9887 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
9888 AssertRC(rc);
9889 pVCpu->hm.s.fClearTrapFlag = false;
9890 pCtx->eflags.Bits.u1TF = 0;
9891 }
9892 /** @todo there seems to be issues with the resume flag when the monitor trap
9893 * flag is pending without being used. Seen early in bios init when
9894 * accessing APIC page in protected mode. */
9895
9896 /*
9897 * Restore VM-exit control settings as we may not reenter this function the
9898 * next time around.
9899 */
9900 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
9901
9902 /* Restore HMCPU indicators. */
9903 pVCpu->hm.s.fUsingDebugLoop = false;
9904 pVCpu->hm.s.fDebugWantRdTscExit = false;
9905 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
9906
9907 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9908 return rcStrict;
9909}
9910
9911
9912/** @} */
9913
9914
9915/**
9916 * Checks if any expensive dtrace probes are enabled and we should go to the
9917 * debug loop.
9918 *
9919 * @returns true if we should use debug loop, false if not.
9920 */
9921static bool hmR0VmxAnyExpensiveProbesEnabled(void)
9922{
9923 /* It's probably faster to OR the raw 32-bit counter variables together.
9924 Since the variables are in an array and the probes are next to one
9925 another (more or less), we have good locality. So, better read
9926 eight-nine cache lines ever time and only have one conditional, than
9927 128+ conditionals, right? */
9928 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
9929 | VBOXVMM_XCPT_DE_ENABLED_RAW()
9930 | VBOXVMM_XCPT_DB_ENABLED_RAW()
9931 | VBOXVMM_XCPT_BP_ENABLED_RAW()
9932 | VBOXVMM_XCPT_OF_ENABLED_RAW()
9933 | VBOXVMM_XCPT_BR_ENABLED_RAW()
9934 | VBOXVMM_XCPT_UD_ENABLED_RAW()
9935 | VBOXVMM_XCPT_NM_ENABLED_RAW()
9936 | VBOXVMM_XCPT_DF_ENABLED_RAW()
9937 | VBOXVMM_XCPT_TS_ENABLED_RAW()
9938 | VBOXVMM_XCPT_NP_ENABLED_RAW()
9939 | VBOXVMM_XCPT_SS_ENABLED_RAW()
9940 | VBOXVMM_XCPT_GP_ENABLED_RAW()
9941 | VBOXVMM_XCPT_PF_ENABLED_RAW()
9942 | VBOXVMM_XCPT_MF_ENABLED_RAW()
9943 | VBOXVMM_XCPT_AC_ENABLED_RAW()
9944 | VBOXVMM_XCPT_XF_ENABLED_RAW()
9945 | VBOXVMM_XCPT_VE_ENABLED_RAW()
9946 | VBOXVMM_XCPT_SX_ENABLED_RAW()
9947 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
9948 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
9949 ) != 0
9950 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
9951 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
9952 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
9953 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
9954 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
9955 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
9956 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
9957 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
9958 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
9959 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
9960 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
9961 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
9962 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
9963 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
9964 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
9965 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
9966 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
9967 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
9968 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
9969 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
9970 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
9971 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
9972 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
9973 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
9974 | VBOXVMM_INSTR_STR_ENABLED_RAW()
9975 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
9976 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
9977 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
9978 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
9979 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
9980 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
9981 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
9982 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
9983 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
9984 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
9985 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
9986 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
9987 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
9988 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
9989 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
9990 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
9991 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
9992 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
9993 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
9994 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
9995 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
9996 ) != 0
9997 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
9998 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
9999 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10000 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10001 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10002 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10003 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10004 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10005 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10006 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10007 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10008 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10009 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10010 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10011 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10012 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10013 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10014 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10015 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10016 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10017 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10018 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10019 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10020 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10021 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10022 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10023 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10024 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10025 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10026 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10027 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10028 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10029 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10030 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10031 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10032 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10033 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10034 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10035 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10036 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10037 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10038 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10039 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10040 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10041 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10042 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10043 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10044 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10045 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10046 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10047 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10048 ) != 0;
10049}
10050
10051
10052/**
10053 * Runs the guest code using VT-x.
10054 *
10055 * @returns Strict VBox status code (i.e. informational status codes too).
10056 * @param pVM The cross context VM structure.
10057 * @param pVCpu The cross context virtual CPU structure.
10058 * @param pCtx Pointer to the guest-CPU context.
10059 */
10060VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10061{
10062 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10063 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10064 HMVMX_ASSERT_PREEMPT_SAFE();
10065
10066 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10067
10068 VBOXSTRICTRC rcStrict;
10069 if ( !pVCpu->hm.s.fUseDebugLoop
10070 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10071 && !DBGFIsStepping(pVCpu)
10072 && !pVM->dbgf.ro.cEnabledInt3Breakpoints)
10073 rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10074 else
10075 rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10076
10077 if (rcStrict == VERR_EM_INTERPRETER)
10078 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10079 else if (rcStrict == VINF_EM_RESET)
10080 rcStrict = VINF_EM_TRIPLE_FAULT;
10081
10082 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10083 if (RT_FAILURE(rc2))
10084 {
10085 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10086 rcStrict = rc2;
10087 }
10088 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10089 return rcStrict;
10090}
10091
10092
10093#ifndef HMVMX_USE_FUNCTION_TABLE
10094DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10095{
10096#ifdef DEBUG_ramshankar
10097#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10098 do { \
10099 if (a_fSave != 0) \
10100 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10101 VBOXSTRICTRC rcStrict = a_CallExpr; \
10102 if (a_fSave != 0) \
10103 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10104 return rcStrict; \
10105 } while (0)
10106#else
10107# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10108#endif
10109 switch (rcReason)
10110 {
10111 case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10112 case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10113 case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10114 case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10115 case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10116 case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10117 case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10118 case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10119 case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10120 case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10121 case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10122 case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10123 case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10124 case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10125 case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10126 case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10127 case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10128 case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10129 case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10130 case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10131 case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10132 case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10133 case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10134 case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10135 case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10136 case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10137 case VMX_EXIT_XDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10138 case VMX_EXIT_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10139 case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10140 case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10141 case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10142 case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10143 case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10144 case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10145
10146 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10147 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10148 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10149 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10150 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10151 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10152 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10153 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10154 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10155
10156 case VMX_EXIT_VMCLEAR:
10157 case VMX_EXIT_VMLAUNCH:
10158 case VMX_EXIT_VMPTRLD:
10159 case VMX_EXIT_VMPTRST:
10160 case VMX_EXIT_VMREAD:
10161 case VMX_EXIT_VMRESUME:
10162 case VMX_EXIT_VMWRITE:
10163 case VMX_EXIT_VMXOFF:
10164 case VMX_EXIT_VMXON:
10165 case VMX_EXIT_INVEPT:
10166 case VMX_EXIT_INVVPID:
10167 case VMX_EXIT_VMFUNC:
10168 case VMX_EXIT_XSAVES:
10169 case VMX_EXIT_XRSTORS:
10170 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10171
10172 case VMX_EXIT_ENCLS:
10173 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10174 case VMX_EXIT_PML_FULL:
10175 default:
10176 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10177 }
10178#undef VMEXIT_CALL_RET
10179}
10180#endif /* !HMVMX_USE_FUNCTION_TABLE */
10181
10182
10183#ifdef VBOX_STRICT
10184/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10185# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10186 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10187
10188# define HMVMX_ASSERT_PREEMPT_CPUID() \
10189 do { \
10190 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10191 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10192 } while (0)
10193
10194# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10195 do { \
10196 AssertPtr(pVCpu); \
10197 AssertPtr(pMixedCtx); \
10198 AssertPtr(pVmxTransient); \
10199 Assert(pVmxTransient->fVMEntryFailed == false); \
10200 Assert(ASMIntAreEnabled()); \
10201 HMVMX_ASSERT_PREEMPT_SAFE(); \
10202 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10203 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10204 HMVMX_ASSERT_PREEMPT_SAFE(); \
10205 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10206 HMVMX_ASSERT_PREEMPT_CPUID(); \
10207 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10208 } while (0)
10209
10210# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10211 do { \
10212 Log4Func(("\n")); \
10213 } while (0)
10214#else /* nonstrict builds: */
10215# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10216 do { \
10217 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10218 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10219 } while (0)
10220# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10221#endif
10222
10223
10224/**
10225 * Advances the guest RIP by the specified number of bytes.
10226 *
10227 * @param pVCpu The cross context virtual CPU structure.
10228 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10229 * out-of-sync. Make sure to update the required fields
10230 * before using them.
10231 * @param cbInstr Number of bytes to advance the RIP by.
10232 *
10233 * @remarks No-long-jump zone!!!
10234 */
10235DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10236{
10237 /* Advance the RIP. */
10238 pMixedCtx->rip += cbInstr;
10239 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10240
10241 /* Update interrupt inhibition. */
10242 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10243 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10244 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10245}
10246
10247
10248/**
10249 * Advances the guest RIP after reading it from the VMCS.
10250 *
10251 * @returns VBox status code, no informational status codes.
10252 * @param pVCpu The cross context virtual CPU structure.
10253 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10254 * out-of-sync. Make sure to update the required fields
10255 * before using them.
10256 * @param pVmxTransient Pointer to the VMX transient structure.
10257 *
10258 * @remarks No-long-jump zone!!!
10259 */
10260static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10261{
10262 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10263 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
10264 | CPUMCTX_EXTRN_RFLAGS);
10265 AssertRCReturn(rc, rc);
10266
10267 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10268
10269 /*
10270 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10271 * pending debug exception field as it takes care of priority of events.
10272 *
10273 * See Intel spec. 32.2.1 "Debug Exceptions".
10274 */
10275 if ( !pVCpu->hm.s.fSingleInstruction
10276 && pMixedCtx->eflags.Bits.u1TF)
10277 {
10278 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
10279 AssertRCReturn(rc, rc);
10280 }
10281
10282 return VINF_SUCCESS;
10283}
10284
10285
10286/**
10287 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10288 * and update error record fields accordingly.
10289 *
10290 * @return VMX_IGS_* return codes.
10291 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10292 * wrong with the guest state.
10293 *
10294 * @param pVM The cross context VM structure.
10295 * @param pVCpu The cross context virtual CPU structure.
10296 * @param pCtx Pointer to the guest-CPU state.
10297 *
10298 * @remarks This function assumes our cache of the VMCS controls
10299 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10300 */
10301static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10302{
10303#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10304#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10305 uError = (err); \
10306 break; \
10307 } else do { } while (0)
10308
10309 int rc;
10310 uint32_t uError = VMX_IGS_ERROR;
10311 uint32_t u32Val;
10312 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10313
10314 do
10315 {
10316 /*
10317 * CR0.
10318 */
10319 uint32_t fSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10320 uint32_t fZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10321 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10322 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10323 if (fUnrestrictedGuest)
10324 fSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
10325
10326 uint32_t uGuestCR0;
10327 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uGuestCR0);
10328 AssertRCBreak(rc);
10329 HMVMX_CHECK_BREAK((uGuestCR0 & fSetCR0) == fSetCR0, VMX_IGS_CR0_FIXED1);
10330 HMVMX_CHECK_BREAK(!(uGuestCR0 & ~fZapCR0), VMX_IGS_CR0_FIXED0);
10331 if ( !fUnrestrictedGuest
10332 && (uGuestCR0 & X86_CR0_PG)
10333 && !(uGuestCR0 & X86_CR0_PE))
10334 {
10335 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10336 }
10337
10338 /*
10339 * CR4.
10340 */
10341 uint64_t fSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10342 uint64_t fZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10343
10344 uint32_t uGuestCR4;
10345 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uGuestCR4);
10346 AssertRCBreak(rc);
10347 HMVMX_CHECK_BREAK((uGuestCR4 & fSetCR4) == fSetCR4, VMX_IGS_CR4_FIXED1);
10348 HMVMX_CHECK_BREAK(!(uGuestCR4 & ~fZapCR4), VMX_IGS_CR4_FIXED0);
10349
10350 /*
10351 * IA32_DEBUGCTL MSR.
10352 */
10353 uint64_t u64Val;
10354 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10355 AssertRCBreak(rc);
10356 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10357 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10358 {
10359 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10360 }
10361 uint64_t u64DebugCtlMsr = u64Val;
10362
10363#ifdef VBOX_STRICT
10364 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10365 AssertRCBreak(rc);
10366 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10367#endif
10368 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10369
10370 /*
10371 * RIP and RFLAGS.
10372 */
10373 uint32_t u32Eflags;
10374#if HC_ARCH_BITS == 64
10375 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10376 AssertRCBreak(rc);
10377 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10378 if ( !fLongModeGuest
10379 || !pCtx->cs.Attr.n.u1Long)
10380 {
10381 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10382 }
10383 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10384 * must be identical if the "IA-32e mode guest" VM-entry
10385 * control is 1 and CS.L is 1. No check applies if the
10386 * CPU supports 64 linear-address bits. */
10387
10388 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10389 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10390 AssertRCBreak(rc);
10391 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10392 VMX_IGS_RFLAGS_RESERVED);
10393 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10394 u32Eflags = u64Val;
10395#else
10396 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10397 AssertRCBreak(rc);
10398 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10399 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10400#endif
10401
10402 if ( fLongModeGuest
10403 || ( fUnrestrictedGuest
10404 && !(uGuestCR0 & X86_CR0_PE)))
10405 {
10406 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10407 }
10408
10409 uint32_t u32EntryInfo;
10410 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10411 AssertRCBreak(rc);
10412 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10413 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10414 {
10415 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10416 }
10417
10418 /*
10419 * 64-bit checks.
10420 */
10421#if HC_ARCH_BITS == 64
10422 if (fLongModeGuest)
10423 {
10424 HMVMX_CHECK_BREAK(uGuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10425 HMVMX_CHECK_BREAK(uGuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10426 }
10427
10428 if ( !fLongModeGuest
10429 && (uGuestCR4 & X86_CR4_PCIDE))
10430 {
10431 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10432 }
10433
10434 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10435 * 51:32 beyond the processor's physical-address width are 0. */
10436
10437 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10438 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10439 {
10440 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10441 }
10442
10443 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10444 AssertRCBreak(rc);
10445 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10446
10447 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10448 AssertRCBreak(rc);
10449 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10450#endif
10451
10452 /*
10453 * PERF_GLOBAL MSR.
10454 */
10455 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10456 {
10457 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10458 AssertRCBreak(rc);
10459 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10460 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10461 }
10462
10463 /*
10464 * PAT MSR.
10465 */
10466 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10467 {
10468 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10469 AssertRCBreak(rc);
10470 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10471 for (unsigned i = 0; i < 8; i++)
10472 {
10473 uint8_t u8Val = (u64Val & 0xff);
10474 if ( u8Val != 0 /* UC */
10475 && u8Val != 1 /* WC */
10476 && u8Val != 4 /* WT */
10477 && u8Val != 5 /* WP */
10478 && u8Val != 6 /* WB */
10479 && u8Val != 7 /* UC- */)
10480 {
10481 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10482 }
10483 u64Val >>= 8;
10484 }
10485 }
10486
10487 /*
10488 * EFER MSR.
10489 */
10490 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10491 {
10492 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10493 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10494 AssertRCBreak(rc);
10495 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10496 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10497 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10498 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10499 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10500 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10501 || !(uGuestCR0 & X86_CR0_PG)
10502 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10503 VMX_IGS_EFER_LMA_LME_MISMATCH);
10504 }
10505
10506 /*
10507 * Segment registers.
10508 */
10509 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10510 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10511 if (!(u32Eflags & X86_EFL_VM))
10512 {
10513 /* CS */
10514 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10515 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10516 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10517 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10518 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10519 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10520 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10521 /* CS cannot be loaded with NULL in protected mode. */
10522 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10523 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10524 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10525 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10526 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10527 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10528 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10529 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10530 else
10531 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10532
10533 /* SS */
10534 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10535 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10536 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10537 if ( !(pCtx->cr0 & X86_CR0_PE)
10538 || pCtx->cs.Attr.n.u4Type == 3)
10539 {
10540 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10541 }
10542 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10543 {
10544 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10545 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10546 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10547 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10548 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10549 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10550 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10551 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10552 }
10553
10554 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
10555 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10556 {
10557 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10558 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10559 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10560 || pCtx->ds.Attr.n.u4Type > 11
10561 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10562 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10563 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10564 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10565 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10566 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10567 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10568 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10569 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10570 }
10571 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10572 {
10573 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10574 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10575 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10576 || pCtx->es.Attr.n.u4Type > 11
10577 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10578 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10579 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10580 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10581 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10582 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10583 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10584 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10585 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10586 }
10587 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10588 {
10589 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10590 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10591 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10592 || pCtx->fs.Attr.n.u4Type > 11
10593 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10594 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10595 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10596 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10597 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10598 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10599 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10600 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10601 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10602 }
10603 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10604 {
10605 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10606 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10607 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10608 || pCtx->gs.Attr.n.u4Type > 11
10609 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10610 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10611 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10612 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10613 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10614 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10615 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10616 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10617 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10618 }
10619 /* 64-bit capable CPUs. */
10620#if HC_ARCH_BITS == 64
10621 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10622 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10623 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10624 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10625 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10626 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10627 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10628 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10629 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10630 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10631 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10632#endif
10633 }
10634 else
10635 {
10636 /* V86 mode checks. */
10637 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10638 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10639 {
10640 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10641 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10642 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10643 }
10644 else
10645 {
10646 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10647 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10648 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10649 }
10650
10651 /* CS */
10652 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10653 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10654 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10655 /* SS */
10656 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10657 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10658 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10659 /* DS */
10660 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10661 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10662 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10663 /* ES */
10664 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10665 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10666 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10667 /* FS */
10668 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10669 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10670 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10671 /* GS */
10672 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10673 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10674 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10675 /* 64-bit capable CPUs. */
10676#if HC_ARCH_BITS == 64
10677 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10678 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10679 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10680 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10681 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10682 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10683 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10684 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10685 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10686 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10687 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10688#endif
10689 }
10690
10691 /*
10692 * TR.
10693 */
10694 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10695 /* 64-bit capable CPUs. */
10696#if HC_ARCH_BITS == 64
10697 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10698#endif
10699 if (fLongModeGuest)
10700 {
10701 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10702 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10703 }
10704 else
10705 {
10706 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10707 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10708 VMX_IGS_TR_ATTR_TYPE_INVALID);
10709 }
10710 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10711 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10712 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10713 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10714 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10715 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10716 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10717 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10718
10719 /*
10720 * GDTR and IDTR.
10721 */
10722#if HC_ARCH_BITS == 64
10723 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10724 AssertRCBreak(rc);
10725 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10726
10727 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10728 AssertRCBreak(rc);
10729 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10730#endif
10731
10732 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10733 AssertRCBreak(rc);
10734 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10735
10736 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10737 AssertRCBreak(rc);
10738 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10739
10740 /*
10741 * Guest Non-Register State.
10742 */
10743 /* Activity State. */
10744 uint32_t u32ActivityState;
10745 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10746 AssertRCBreak(rc);
10747 HMVMX_CHECK_BREAK( !u32ActivityState
10748 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
10749 VMX_IGS_ACTIVITY_STATE_INVALID);
10750 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10751 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10752 uint32_t u32IntrState;
10753 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
10754 AssertRCBreak(rc);
10755 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
10756 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10757 {
10758 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10759 }
10760
10761 /** @todo Activity state and injecting interrupts. Left as a todo since we
10762 * currently don't use activity states but ACTIVE. */
10763
10764 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10765 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10766
10767 /* Guest interruptibility-state. */
10768 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10769 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10770 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
10771 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10772 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10773 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10774 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10775 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10776 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10777 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
10778 {
10779 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10780 {
10781 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10782 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10783 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10784 }
10785 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10786 {
10787 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10788 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10789 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10790 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10791 }
10792 }
10793 /** @todo Assumes the processor is not in SMM. */
10794 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10795 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10796 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10797 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10798 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10799 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
10800 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10801 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10802 {
10803 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
10804 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10805 }
10806
10807 /* Pending debug exceptions. */
10808#if HC_ARCH_BITS == 64
10809 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
10810 AssertRCBreak(rc);
10811 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10812 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10813 u32Val = u64Val; /* For pending debug exceptions checks below. */
10814#else
10815 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
10816 AssertRCBreak(rc);
10817 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10818 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10819#endif
10820
10821 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10822 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
10823 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10824 {
10825 if ( (u32Eflags & X86_EFL_TF)
10826 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10827 {
10828 /* Bit 14 is PendingDebug.BS. */
10829 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10830 }
10831 if ( !(u32Eflags & X86_EFL_TF)
10832 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10833 {
10834 /* Bit 14 is PendingDebug.BS. */
10835 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10836 }
10837 }
10838
10839 /* VMCS link pointer. */
10840 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10841 AssertRCBreak(rc);
10842 if (u64Val != UINT64_C(0xffffffffffffffff))
10843 {
10844 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10845 /** @todo Bits beyond the processor's physical-address width MBZ. */
10846 /** @todo 32-bit located in memory referenced by value of this field (as a
10847 * physical address) must contain the processor's VMCS revision ID. */
10848 /** @todo SMM checks. */
10849 }
10850
10851 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10852 * not using Nested Paging? */
10853 if ( pVM->hm.s.fNestedPaging
10854 && !fLongModeGuest
10855 && CPUMIsGuestInPAEModeEx(pCtx))
10856 {
10857 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10858 AssertRCBreak(rc);
10859 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10860
10861 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10862 AssertRCBreak(rc);
10863 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10864
10865 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10866 AssertRCBreak(rc);
10867 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10868
10869 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10870 AssertRCBreak(rc);
10871 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10872 }
10873
10874 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10875 if (uError == VMX_IGS_ERROR)
10876 uError = VMX_IGS_REASON_NOT_FOUND;
10877 } while (0);
10878
10879 pVCpu->hm.s.u32HMError = uError;
10880 return uError;
10881
10882#undef HMVMX_ERROR_BREAK
10883#undef HMVMX_CHECK_BREAK
10884}
10885
10886/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10887/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
10888/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10889
10890/** @name VM-exit handlers.
10891 * @{
10892 */
10893
10894/**
10895 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
10896 */
10897HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10898{
10899 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10900 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
10901 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
10902 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
10903 return VINF_SUCCESS;
10904 return VINF_EM_RAW_INTERRUPT;
10905}
10906
10907
10908/**
10909 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
10910 */
10911HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10912{
10913 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10914 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
10915
10916 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10917 AssertRCReturn(rc, rc);
10918
10919 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10920 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
10921 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
10922 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
10923
10924 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10925 {
10926 /*
10927 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
10928 * injected it ourselves and anything we inject is not going to cause a VM-exit directly
10929 * for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
10930 *
10931 * [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
10932 * [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
10933 */
10934 VMXDispatchHostNmi();
10935 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
10936 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10937 return VINF_SUCCESS;
10938 }
10939
10940 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10941 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10942 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
10943 { /* likely */ }
10944 else
10945 {
10946 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
10947 rcStrictRc1 = VINF_SUCCESS;
10948 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10949 return rcStrictRc1;
10950 }
10951
10952 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
10953 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
10954 switch (uIntType)
10955 {
10956 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
10957 Assert(uVector == X86_XCPT_DB);
10958 RT_FALL_THRU();
10959 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
10960 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
10961 RT_FALL_THRU();
10962 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
10963 {
10964 /*
10965 * If there's any exception caused as a result of event injection, the resulting
10966 * secondary/final execption will be pending, we shall continue guest execution
10967 * after injecting the event. The page-fault case is complicated and we manually
10968 * handle any currently pending event in hmR0VmxExitXcptPF.
10969 */
10970 if (!pVCpu->hm.s.Event.fPending)
10971 { /* likely */ }
10972 else if (uVector != X86_XCPT_PF)
10973 {
10974 rc = VINF_SUCCESS;
10975 break;
10976 }
10977
10978 switch (uVector)
10979 {
10980 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
10981 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
10982 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
10983 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
10984 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
10985 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
10986
10987 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10988 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10989 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
10990 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10991 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
10992 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10993 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
10994 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10995 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
10996 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10997 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
10998 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10999 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11000 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11001 default:
11002 {
11003 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11004 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11005 {
11006 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11007 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11008 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11009
11010 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
11011 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11012 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11013 AssertRCReturn(rc, rc);
11014 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11015 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11016 0 /* GCPtrFaultAddress */);
11017 }
11018 else
11019 {
11020 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11021 pVCpu->hm.s.u32HMError = uVector;
11022 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11023 }
11024 break;
11025 }
11026 }
11027 break;
11028 }
11029
11030 default:
11031 {
11032 pVCpu->hm.s.u32HMError = uExitIntInfo;
11033 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11034 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11035 break;
11036 }
11037 }
11038 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11039 return rc;
11040}
11041
11042
11043/**
11044 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11045 */
11046HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11047{
11048 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11049
11050 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11051 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11052
11053 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11054 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11055 return VINF_SUCCESS;
11056}
11057
11058
11059/**
11060 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11061 */
11062HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11063{
11064 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11065 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11066 {
11067 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11068 HMVMX_RETURN_UNEXPECTED_EXIT();
11069 }
11070
11071 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11072
11073 /*
11074 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11075 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11076 */
11077 uint32_t fIntrState = 0;
11078 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11079 AssertRCReturn(rc, rc);
11080
11081 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11082 if ( fBlockSti
11083 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11084 {
11085 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11086 }
11087
11088 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11089 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11090
11091 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11092 return VINF_SUCCESS;
11093}
11094
11095
11096/**
11097 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11098 */
11099HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11100{
11101 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11102 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11103}
11104
11105
11106/**
11107 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11108 */
11109HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11110{
11111 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11112 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11113}
11114
11115
11116/**
11117 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11118 */
11119HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11120{
11121 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11122 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
11123
11124 /*
11125 * Get the state we need and update the exit history entry.
11126 */
11127 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11128 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
11129 | CPUMCTX_EXTRN_CS);
11130 AssertRCReturn(rc, rc);
11131
11132 VBOXSTRICTRC rcStrict;
11133 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11134 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
11135 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11136 if (!pExitRec)
11137 {
11138 /*
11139 * Regular CPUID instruction execution.
11140 */
11141 PVM pVM = pVCpu->CTX_SUFF(pVM);
11142 rcStrict = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11143 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11144 {
11145 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11146 Assert(pVmxTransient->cbInstr == 2);
11147 }
11148 else
11149 {
11150 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11151 rcStrict = VERR_EM_INTERPRETER;
11152 }
11153 }
11154 else
11155 {
11156 /*
11157 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
11158 */
11159 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
11160 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11161 AssertRCReturn(rc2, rc2);
11162
11163 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11164 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11165
11166 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11167 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11168
11169 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11170 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11171 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11172 }
11173 return VBOXSTRICTRC_TODO(rcStrict);
11174}
11175
11176
11177/**
11178 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11179 */
11180HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11181{
11182 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11183 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4);
11184 AssertRCReturn(rc, rc);
11185
11186 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11187 return VINF_EM_RAW_EMULATE_INSTR;
11188
11189 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11190 HMVMX_RETURN_UNEXPECTED_EXIT();
11191}
11192
11193
11194/**
11195 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11196 */
11197HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11198{
11199 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11200 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11201 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11202 AssertRCReturn(rc, rc);
11203
11204 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11205 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11206 {
11207 /* If we get a spurious VM-exit when offsetting is enabled,
11208 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11209 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11210 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11211 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
11212 | HM_CHANGED_GUEST_RFLAGS);
11213 }
11214 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11215 {
11216 rcStrict = VINF_SUCCESS;
11217 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11218 }
11219 return rcStrict;
11220}
11221
11222
11223/**
11224 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11225 */
11226HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11227{
11228 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11229 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
11230 | CPUMCTX_EXTRN_TSC_AUX);
11231 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11232 AssertRCReturn(rc, rc);
11233
11234 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11235 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11236 {
11237 /* If we get a spurious VM-exit when offsetting is enabled,
11238 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11239 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11240 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11241 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
11242 | HM_CHANGED_GUEST_RFLAGS);
11243 }
11244 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11245 {
11246 rcStrict = VINF_SUCCESS;
11247 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11248 }
11249 return rcStrict;
11250}
11251
11252
11253/**
11254 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11255 */
11256HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11257{
11258 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11259 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4
11260 | CPUMCTX_EXTRN_CR0
11261 | CPUMCTX_EXTRN_RFLAGS
11262 | CPUMCTX_EXTRN_SS);
11263 AssertRCReturn(rc, rc);
11264
11265 PVM pVM = pVCpu->CTX_SUFF(pVM);
11266 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11267 if (RT_LIKELY(rc == VINF_SUCCESS))
11268 {
11269 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11270 Assert(pVmxTransient->cbInstr == 2);
11271 }
11272 else
11273 {
11274 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11275 rc = VERR_EM_INTERPRETER;
11276 }
11277 return rc;
11278}
11279
11280
11281/**
11282 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11283 */
11284HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11285{
11286 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11287
11288 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11289 if (EMAreHypercallInstructionsEnabled(pVCpu))
11290 {
11291 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
11292 | CPUMCTX_EXTRN_RFLAGS
11293 | CPUMCTX_EXTRN_CR0
11294 | CPUMCTX_EXTRN_SS
11295 | CPUMCTX_EXTRN_CS
11296 | CPUMCTX_EXTRN_EFER);
11297 AssertRCReturn(rc, rc);
11298
11299 /* Perform the hypercall. */
11300 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11301 if (rcStrict == VINF_SUCCESS)
11302 {
11303 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11304 AssertRCReturn(rc, rc);
11305 }
11306 else
11307 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11308 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11309 || RT_FAILURE(rcStrict));
11310
11311 /* If the hypercall changes anything other than guest's general-purpose registers,
11312 we would need to reload the guest changed bits here before VM-entry. */
11313 }
11314 else
11315 Log4Func(("Hypercalls not enabled\n"));
11316
11317 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11318 if (RT_FAILURE(rcStrict))
11319 {
11320 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11321 rcStrict = VINF_SUCCESS;
11322 }
11323
11324 return rcStrict;
11325}
11326
11327
11328/**
11329 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11330 */
11331HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11332{
11333 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11334 PVM pVM = pVCpu->CTX_SUFF(pVM);
11335 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11336
11337 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11338 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK);
11339 AssertRCReturn(rc, rc);
11340
11341 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11342 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11343 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11344 else
11345 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11346 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11347 return rcStrict;
11348}
11349
11350
11351/**
11352 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11353 */
11354HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11355{
11356 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11357 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11358 | CPUMCTX_EXTRN_RFLAGS
11359 | CPUMCTX_EXTRN_SS);
11360 AssertRCReturn(rc, rc);
11361
11362 PVM pVM = pVCpu->CTX_SUFF(pVM);
11363 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11364 if (RT_LIKELY(rc == VINF_SUCCESS))
11365 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11366 else
11367 {
11368 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11369 rc = VERR_EM_INTERPRETER;
11370 }
11371 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11372 return rc;
11373}
11374
11375
11376/**
11377 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11378 */
11379HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11380{
11381 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11382 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11383 | CPUMCTX_EXTRN_RFLAGS
11384 | CPUMCTX_EXTRN_SS);
11385 AssertRCReturn(rc, rc);
11386
11387 PVM pVM = pVCpu->CTX_SUFF(pVM);
11388 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11389 rc = VBOXSTRICTRC_VAL(rc2);
11390 if (RT_LIKELY( rc == VINF_SUCCESS
11391 || rc == VINF_EM_HALT))
11392 {
11393 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11394 AssertRCReturn(rc3, rc3);
11395
11396 if ( rc == VINF_EM_HALT
11397 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11398 rc = VINF_SUCCESS;
11399 }
11400 else
11401 {
11402 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11403 rc = VERR_EM_INTERPRETER;
11404 }
11405 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11406 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11407 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11408 return rc;
11409}
11410
11411
11412/**
11413 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11414 */
11415HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11416{
11417 /*
11418 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11419 * mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11420 * treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11421 * VMX root operation. If we get here, something funny is going on.
11422 *
11423 * See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11424 */
11425 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11426 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11427 HMVMX_RETURN_UNEXPECTED_EXIT();
11428}
11429
11430
11431/**
11432 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11433 */
11434HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11435{
11436 /*
11437 * This can only happen if we support dual-monitor treatment of SMI, which can be activated
11438 * by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11439 * this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11440 * an SMI. If we get here, something funny is going on.
11441 *
11442 * See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11443 * See Intel spec. 25.3 "Other Causes of VM-Exits"
11444 */
11445 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11446 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11447 HMVMX_RETURN_UNEXPECTED_EXIT();
11448}
11449
11450
11451/**
11452 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11453 */
11454HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11455{
11456 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11457 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11458 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11459 HMVMX_RETURN_UNEXPECTED_EXIT();
11460}
11461
11462
11463/**
11464 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11465 */
11466HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11467{
11468 /*
11469 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11470 * We don't make use of it as our guests don't have direct access to the host LAPIC.
11471 * See Intel spec. 25.3 "Other Causes of VM-exits".
11472 */
11473 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11474 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11475 HMVMX_RETURN_UNEXPECTED_EXIT();
11476}
11477
11478
11479/**
11480 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11481 * VM-exit.
11482 */
11483HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11484{
11485 /*
11486 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11487 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11488 *
11489 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11490 * See Intel spec. "23.8 Restrictions on VMX operation".
11491 */
11492 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11493 return VINF_SUCCESS;
11494}
11495
11496
11497/**
11498 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11499 * VM-exit.
11500 */
11501HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11502{
11503 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11504 return VINF_EM_RESET;
11505}
11506
11507
11508/**
11509 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11510 */
11511HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11512{
11513 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11514 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11515
11516 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11517 AssertRCReturn(rc, rc);
11518
11519 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11520 rc = VINF_SUCCESS;
11521 else
11522 rc = VINF_EM_HALT;
11523
11524 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11525 if (rc != VINF_SUCCESS)
11526 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11527 return rc;
11528}
11529
11530
11531/**
11532 * VM-exit handler for instructions that result in a \#UD exception delivered to
11533 * the guest.
11534 */
11535HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11536{
11537 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11538 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11539 return VINF_SUCCESS;
11540}
11541
11542
11543/**
11544 * VM-exit handler for expiry of the VMX preemption timer.
11545 */
11546HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11547{
11548 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11549
11550 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11551 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11552
11553 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11554 PVM pVM = pVCpu->CTX_SUFF(pVM);
11555 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11556 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11557 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11558}
11559
11560
11561/**
11562 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11563 */
11564HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11565{
11566 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11567
11568 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11569 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
11570 | CPUMCTX_EXTRN_CR4);
11571 AssertRCReturn(rc, rc);
11572
11573 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11574 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11575 : HM_CHANGED_XCPT_RAISED_MASK);
11576
11577 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11578
11579 return rcStrict;
11580}
11581
11582
11583/**
11584 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11585 */
11586HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11587{
11588 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11589 /** @todo Use VM-exit instruction information. */
11590 return VERR_EM_INTERPRETER;
11591}
11592
11593
11594/**
11595 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11596 * Error VM-exit.
11597 */
11598HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11599{
11600 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11601 rc |= hmR0VmxCheckVmcsCtls(pVCpu);
11602 AssertRCReturn(rc, rc);
11603
11604 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11605 NOREF(uInvalidReason);
11606
11607#ifdef VBOX_STRICT
11608 uint32_t fIntrState;
11609 RTHCUINTREG uHCReg;
11610 uint64_t u64Val;
11611 uint32_t u32Val;
11612
11613 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11614 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11615 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11616 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11617 AssertRCReturn(rc, rc);
11618
11619 Log4(("uInvalidReason %u\n", uInvalidReason));
11620 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11621 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11622 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11623 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", fIntrState));
11624
11625 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11626 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11627 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11628 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11629 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11630 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11631 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11632 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11633 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11634 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11635 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11636 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11637#else
11638 NOREF(pVmxTransient);
11639#endif
11640
11641 hmR0DumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11642 return VERR_VMX_INVALID_GUEST_STATE;
11643}
11644
11645
11646/**
11647 * VM-exit handler for VM-entry failure due to an MSR-load
11648 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11649 */
11650HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11651{
11652 NOREF(pVmxTransient);
11653 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11654 HMVMX_RETURN_UNEXPECTED_EXIT();
11655}
11656
11657
11658/**
11659 * VM-exit handler for VM-entry failure due to a machine-check event
11660 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11661 */
11662HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11663{
11664 NOREF(pVmxTransient);
11665 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11666 HMVMX_RETURN_UNEXPECTED_EXIT();
11667}
11668
11669
11670/**
11671 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11672 * theory.
11673 */
11674HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11675{
11676 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
11677 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
11678 return VERR_VMX_UNDEFINED_EXIT_CODE;
11679}
11680
11681
11682/**
11683 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11684 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11685 * Conditional VM-exit.
11686 */
11687HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11688{
11689 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11690
11691 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11692 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11693 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11694 return VERR_EM_INTERPRETER;
11695 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11696 HMVMX_RETURN_UNEXPECTED_EXIT();
11697}
11698
11699
11700/**
11701 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11702 */
11703HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11704{
11705 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11706
11707 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11708 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11709 return VERR_EM_INTERPRETER;
11710 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11711 HMVMX_RETURN_UNEXPECTED_EXIT();
11712}
11713
11714
11715/**
11716 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11717 */
11718HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11719{
11720 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11721
11722 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
11723 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11724 | CPUMCTX_EXTRN_RFLAGS
11725 | CPUMCTX_EXTRN_SS);
11726 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11727 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
11728 AssertRCReturn(rc, rc);
11729 Log4Func(("ecx=%#RX32\n", pMixedCtx->ecx));
11730
11731#ifdef VBOX_STRICT
11732 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
11733 {
11734 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
11735 && pMixedCtx->ecx != MSR_K6_EFER)
11736 {
11737 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11738 pMixedCtx->ecx));
11739 HMVMX_RETURN_UNEXPECTED_EXIT();
11740 }
11741 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11742 {
11743 VMXMSREXITREAD enmRead;
11744 VMXMSREXITWRITE enmWrite;
11745 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
11746 AssertRCReturn(rc2, rc2);
11747 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11748 {
11749 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11750 HMVMX_RETURN_UNEXPECTED_EXIT();
11751 }
11752 }
11753 }
11754#endif
11755
11756 PVM pVM = pVCpu->CTX_SUFF(pVM);
11757 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11758 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
11759 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
11760 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11761 if (RT_SUCCESS(rc))
11762 {
11763 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11764 Assert(pVmxTransient->cbInstr == 2);
11765 }
11766 return rc;
11767}
11768
11769
11770/**
11771 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11772 */
11773HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11774{
11775 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11776 PVM pVM = pVCpu->CTX_SUFF(pVM);
11777 int rc = VINF_SUCCESS;
11778
11779 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
11780 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11781 | CPUMCTX_EXTRN_RFLAGS
11782 | CPUMCTX_EXTRN_SS);
11783 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11784 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
11785 AssertRCReturn(rc, rc);
11786 Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
11787
11788 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11789 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
11790 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
11791
11792 if (RT_SUCCESS(rc))
11793 {
11794 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11795
11796 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
11797 if ( pMixedCtx->ecx == MSR_IA32_APICBASE
11798 || ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
11799 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
11800 {
11801 /*
11802 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
11803 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
11804 * EMInterpretWrmsr() changes it.
11805 */
11806 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11807 }
11808 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
11809 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11810 else if (pMixedCtx->ecx == MSR_K6_EFER)
11811 {
11812 /*
11813 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
11814 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
11815 * the other bits as well, SCE and NXE. See @bugref{7368}.
11816 */
11817 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
11818 | HM_CHANGED_VMX_ENTRY_CTLS
11819 | HM_CHANGED_VMX_EXIT_CTLS);
11820 }
11821
11822 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
11823 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11824 {
11825 switch (pMixedCtx->ecx)
11826 {
11827 case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
11828 case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
11829 case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
11830 case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
11831 case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
11832 case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
11833 default:
11834 {
11835 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11836 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
11837 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11838 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
11839 break;
11840 }
11841 }
11842 }
11843#ifdef VBOX_STRICT
11844 else
11845 {
11846 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
11847 switch (pMixedCtx->ecx)
11848 {
11849 case MSR_IA32_SYSENTER_CS:
11850 case MSR_IA32_SYSENTER_EIP:
11851 case MSR_IA32_SYSENTER_ESP:
11852 case MSR_K8_FS_BASE:
11853 case MSR_K8_GS_BASE:
11854 {
11855 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
11856 HMVMX_RETURN_UNEXPECTED_EXIT();
11857 }
11858
11859 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
11860 default:
11861 {
11862 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11863 {
11864 /* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
11865 if (pMixedCtx->ecx != MSR_K6_EFER)
11866 {
11867 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11868 pMixedCtx->ecx));
11869 HMVMX_RETURN_UNEXPECTED_EXIT();
11870 }
11871 }
11872
11873 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11874 {
11875 VMXMSREXITREAD enmRead;
11876 VMXMSREXITWRITE enmWrite;
11877 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
11878 AssertRCReturn(rc2, rc2);
11879 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
11880 {
11881 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11882 HMVMX_RETURN_UNEXPECTED_EXIT();
11883 }
11884 }
11885 break;
11886 }
11887 }
11888 }
11889#endif /* VBOX_STRICT */
11890 }
11891 return rc;
11892}
11893
11894
11895/**
11896 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
11897 */
11898HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11899{
11900 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11901 /** @todo The guest has likely hit a contended spinlock. We might want to
11902 * poke a schedule different guest VCPU. */
11903 return VINF_EM_RAW_INTERRUPT;
11904}
11905
11906
11907/**
11908 * VM-exit handler for when the TPR value is lowered below the specified
11909 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
11910 */
11911HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11912{
11913 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11914 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
11915
11916 /*
11917 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
11918 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
11919 */
11920 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
11921 return VINF_SUCCESS;
11922}
11923
11924
11925/**
11926 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
11927 * VM-exit.
11928 *
11929 * @retval VINF_SUCCESS when guest execution can continue.
11930 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
11931 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
11932 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
11933 * interpreter.
11934 */
11935HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11936{
11937 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11938 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
11939
11940 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11941 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11942 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11943 AssertRCReturn(rc, rc);
11944
11945 VBOXSTRICTRC rcStrict;
11946 PVM pVM = pVCpu->CTX_SUFF(pVM);
11947 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
11948 uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQualification);
11949 switch (uAccessType)
11950 {
11951 case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
11952 {
11953 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
11954 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
11955 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification));
11956 AssertMsg( rcStrict == VINF_SUCCESS
11957 || rcStrict == VINF_IEM_RAISED_XCPT
11958 || rcStrict == VINF_PGM_CHANGE_MODE
11959 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11960
11961 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
11962 {
11963 case 0:
11964 {
11965 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11966 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
11967 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
11968 break;
11969 }
11970
11971 case 2:
11972 {
11973 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
11974 /* Nothing to do here, CR2 it's not part of the VMCS. */
11975 break;
11976 }
11977
11978 case 3:
11979 {
11980 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
11981 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
11982 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR3);
11983 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
11984 break;
11985 }
11986
11987 case 4:
11988 {
11989 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
11990 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11991 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
11992 pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
11993 break;
11994 }
11995
11996 case 8:
11997 {
11998 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
11999 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12000 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
12001 break;
12002 }
12003 default:
12004 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification)));
12005 break;
12006 }
12007 break;
12008 }
12009
12010 case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
12011 {
12012 Assert( !pVM->hm.s.fNestedPaging
12013 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12014 || pVCpu->hm.s.fUsingDebugLoop
12015 || VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 3);
12016 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12017 Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 8
12018 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12019
12020 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12021 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification),
12022 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification));
12023 AssertMsg( rcStrict == VINF_SUCCESS
12024 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12025#ifdef VBOX_WITH_STATISTICS
12026 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
12027 {
12028 case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
12029 case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
12030 case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
12031 case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
12032 case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
12033 }
12034#endif
12035 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
12036 VBOXSTRICTRC_VAL(rcStrict)));
12037 if (VMX_EXIT_QUAL_CRX_GENREG(uExitQualification) == X86_GREG_xSP)
12038 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RSP);
12039 break;
12040 }
12041
12042 case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12043 {
12044 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12045 AssertMsg( rcStrict == VINF_SUCCESS
12046 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12047
12048 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12049 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12050 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12051 break;
12052 }
12053
12054 case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12055 {
12056 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12057 VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQualification));
12058 AssertMsg( rcStrict == VINF_SUCCESS
12059 || rcStrict == VINF_IEM_RAISED_XCPT
12060 || rcStrict == VINF_PGM_CHANGE_MODE,
12061 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12062
12063 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12064 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12065 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12066 break;
12067 }
12068
12069 default:
12070 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12071 VERR_VMX_UNEXPECTED_EXCEPTION);
12072 }
12073
12074 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
12075 : HM_CHANGED_XCPT_RAISED_MASK);
12076 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12077 NOREF(pVM);
12078 return rcStrict;
12079}
12080
12081
12082/**
12083 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12084 * VM-exit.
12085 */
12086HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12087{
12088 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12089 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12090 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
12091
12092 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12093 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12094 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
12095 | CPUMCTX_EXTRN_SREG_MASK
12096 | CPUMCTX_EXTRN_EFER);
12097 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12098 AssertRCReturn(rc, rc);
12099
12100 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12101 uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQualification);
12102 uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQualification);
12103 bool fIOWrite = ( VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQualification)
12104 == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12105 bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQualification);
12106 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12107 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12108 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12109
12110 /*
12111 * Update exit history to see if this exit can be optimized.
12112 */
12113 VBOXSTRICTRC rcStrict;
12114 PCEMEXITREC pExitRec = NULL;
12115 if ( !fGstStepping
12116 && !fDbgStepping)
12117 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12118 !fIOString
12119 ? !fIOWrite
12120 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12121 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12122 : !fIOWrite
12123 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12124 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12125 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12126 if (!pExitRec)
12127 {
12128 /* I/O operation lookup arrays. */
12129 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12130 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12131 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12132 uint32_t const cbInstr = pVmxTransient->cbInstr;
12133 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12134 PVM pVM = pVCpu->CTX_SUFF(pVM);
12135 if (fIOString)
12136 {
12137 /*
12138 * INS/OUTS - I/O String instruction.
12139 *
12140 * Use instruction-information if available, otherwise fall back on
12141 * interpreting the instruction.
12142 */
12143 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12144 fIOWrite ? 'w' : 'r'));
12145 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12146 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12147 {
12148 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12149 AssertRCReturn(rc2, rc2);
12150 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12151 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12152 IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12153 bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification);
12154 if (fIOWrite)
12155 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12156 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12157 else
12158 {
12159 /*
12160 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12161 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12162 * See Intel Instruction spec. for "INS".
12163 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12164 */
12165 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12166 }
12167 }
12168 else
12169 rcStrict = IEMExecOne(pVCpu);
12170
12171 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12172 fUpdateRipAlready = true;
12173 }
12174 else
12175 {
12176 /*
12177 * IN/OUT - I/O instruction.
12178 */
12179 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12180 fIOWrite ? 'w' : 'r'));
12181 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12182 Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification));
12183 if (fIOWrite)
12184 {
12185 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12186 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12187 }
12188 else
12189 {
12190 uint32_t u32Result = 0;
12191 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12192 if (IOM_SUCCESS(rcStrict))
12193 {
12194 /* Save result of I/O IN instr. in AL/AX/EAX. */
12195 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12196 }
12197 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12198 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12199 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12200 }
12201 }
12202
12203 if (IOM_SUCCESS(rcStrict))
12204 {
12205 if (!fUpdateRipAlready)
12206 {
12207 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12208 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12209 }
12210
12211 /*
12212 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12213 * while booting Fedora 17 64-bit guest.
12214 *
12215 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12216 */
12217 if (fIOString)
12218 {
12219 /** @todo Single-step for INS/OUTS with REP prefix? */
12220 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12221 }
12222 else if ( !fDbgStepping
12223 && fGstStepping)
12224 {
12225 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
12226 AssertRCReturn(rc, rc);
12227 }
12228
12229 /*
12230 * If any I/O breakpoints are armed, we need to check if one triggered
12231 * and take appropriate action.
12232 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12233 */
12234 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12235 AssertRCReturn(rc, rc);
12236
12237 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12238 * execution engines about whether hyper BPs and such are pending. */
12239 uint32_t const uDr7 = pMixedCtx->dr[7];
12240 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12241 && X86_DR7_ANY_RW_IO(uDr7)
12242 && (pMixedCtx->cr4 & X86_CR4_DE))
12243 || DBGFBpIsHwIoArmed(pVM)))
12244 {
12245 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12246
12247 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12248 VMMRZCallRing3Disable(pVCpu);
12249 HM_DISABLE_PREEMPT();
12250
12251 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12252
12253 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12254 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12255 {
12256 /* Raise #DB. */
12257 if (fIsGuestDbgActive)
12258 ASMSetDR6(pMixedCtx->dr[6]);
12259 if (pMixedCtx->dr[7] != uDr7)
12260 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR7;
12261
12262 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12263 }
12264 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12265 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12266 else if ( rcStrict2 != VINF_SUCCESS
12267 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12268 rcStrict = rcStrict2;
12269 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12270
12271 HM_RESTORE_PREEMPT();
12272 VMMRZCallRing3Enable(pVCpu);
12273 }
12274 }
12275
12276#ifdef VBOX_STRICT
12277 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12278 Assert(!fIOWrite);
12279 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12280 Assert(fIOWrite);
12281 else
12282 {
12283# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12284 * statuses, that the VMM device and some others may return. See
12285 * IOM_SUCCESS() for guidance. */
12286 AssertMsg( RT_FAILURE(rcStrict)
12287 || rcStrict == VINF_SUCCESS
12288 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12289 || rcStrict == VINF_EM_DBG_BREAKPOINT
12290 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12291 || rcStrict == VINF_EM_RAW_TO_R3
12292 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12293# endif
12294 }
12295#endif
12296 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12297 }
12298 else
12299 {
12300 /*
12301 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12302 */
12303 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12304 AssertRCReturn(rc2, rc2);
12305 STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12306 : fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12307 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12308 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12309 VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification) ? "REP " : "",
12310 fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12311
12312 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12313 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12314
12315 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12316 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12317 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12318 }
12319 return rcStrict;
12320}
12321
12322
12323/**
12324 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12325 * VM-exit.
12326 */
12327HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12328{
12329 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12330
12331 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12332 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12333 AssertRCReturn(rc, rc);
12334 if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12335 {
12336 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12337 AssertRCReturn(rc, rc);
12338 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12339 {
12340 uint32_t uErrCode;
12341 RTGCUINTPTR GCPtrFaultAddress;
12342 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12343 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12344 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12345 if (fErrorCodeValid)
12346 {
12347 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12348 AssertRCReturn(rc, rc);
12349 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12350 }
12351 else
12352 uErrCode = 0;
12353
12354 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12355 && uVector == X86_XCPT_PF)
12356 GCPtrFaultAddress = pMixedCtx->cr2;
12357 else
12358 GCPtrFaultAddress = 0;
12359
12360 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12361 0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12362
12363 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12364 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12365 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12366 }
12367 }
12368
12369 /* Fall back to the interpreter to emulate the task-switch. */
12370 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12371 return VERR_EM_INTERPRETER;
12372}
12373
12374
12375/**
12376 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12377 */
12378HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12379{
12380 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12381 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12382 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12383 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12384 AssertRCReturn(rc, rc);
12385 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12386 return VINF_EM_DBG_STEPPED;
12387}
12388
12389
12390/**
12391 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12392 */
12393HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12394{
12395 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12396
12397 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12398
12399 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12400 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12401 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12402 {
12403 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12404 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12405 {
12406 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12407 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12408 }
12409 }
12410 else
12411 {
12412 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12413 rcStrict1 = VINF_SUCCESS;
12414 return rcStrict1;
12415 }
12416
12417 /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12418 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12419 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12420 AssertRCReturn(rc, rc);
12421
12422 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12423 uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12424 VBOXSTRICTRC rcStrict2;
12425 switch (uAccessType)
12426 {
12427 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12428 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12429 {
12430 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12431 || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
12432 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12433
12434 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12435 GCPhys &= PAGE_BASE_GC_MASK;
12436 GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12437 PVM pVM = pVCpu->CTX_SUFF(pVM);
12438 Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12439 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12440
12441 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12442 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12443 CPUMCTX2CORE(pMixedCtx), GCPhys);
12444 Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12445 if ( rcStrict2 == VINF_SUCCESS
12446 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12447 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12448 {
12449 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12450 | HM_CHANGED_GUEST_RSP
12451 | HM_CHANGED_GUEST_RFLAGS
12452 | HM_CHANGED_GUEST_APIC_TPR);
12453 rcStrict2 = VINF_SUCCESS;
12454 }
12455 break;
12456 }
12457
12458 default:
12459 Log4Func(("uAccessType=%#x\n", uAccessType));
12460 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12461 break;
12462 }
12463
12464 if (rcStrict2 != VINF_SUCCESS)
12465 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12466 return rcStrict2;
12467}
12468
12469
12470/**
12471 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12472 * VM-exit.
12473 */
12474HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12475{
12476 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12477
12478 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12479 if (pVmxTransient->fWasGuestDebugStateActive)
12480 {
12481 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12482 HMVMX_RETURN_UNEXPECTED_EXIT();
12483 }
12484
12485 if ( !pVCpu->hm.s.fSingleInstruction
12486 && !pVmxTransient->fWasHyperDebugStateActive)
12487 {
12488 Assert(!DBGFIsStepping(pVCpu));
12489 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12490
12491 /* Don't intercept MOV DRx any more. */
12492 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12493 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12494 AssertRCReturn(rc, rc);
12495
12496 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12497 VMMRZCallRing3Disable(pVCpu);
12498 HM_DISABLE_PREEMPT();
12499
12500 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12501 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12502 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12503
12504 HM_RESTORE_PREEMPT();
12505 VMMRZCallRing3Enable(pVCpu);
12506
12507#ifdef VBOX_WITH_STATISTICS
12508 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12509 AssertRCReturn(rc, rc);
12510 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12511 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12512 else
12513 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12514#endif
12515 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12516 return VINF_SUCCESS;
12517 }
12518
12519 /*
12520 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12521 * Update the segment registers and DR7 from the CPU.
12522 */
12523 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12524 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK
12525 | CPUMCTX_EXTRN_DR7);
12526 AssertRCReturn(rc, rc);
12527 Log4Func(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12528
12529 PVM pVM = pVCpu->CTX_SUFF(pVM);
12530 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12531 {
12532 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12533 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification),
12534 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification));
12535 if (RT_SUCCESS(rc))
12536 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12537 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12538 }
12539 else
12540 {
12541 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12542 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification),
12543 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification));
12544 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12545 }
12546
12547 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12548 if (RT_SUCCESS(rc))
12549 {
12550 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12551 AssertRCReturn(rc2, rc2);
12552 return VINF_SUCCESS;
12553 }
12554 return rc;
12555}
12556
12557
12558/**
12559 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12560 * Conditional VM-exit.
12561 */
12562HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12563{
12564 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12565 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12566
12567 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12568 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12569 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12570 {
12571 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12572 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12573 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12574 {
12575 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12576 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12577 }
12578 }
12579 else
12580 {
12581 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12582 rcStrict1 = VINF_SUCCESS;
12583 return rcStrict1;
12584 }
12585
12586 /*
12587 * Get sufficent state and update the exit history entry.
12588 */
12589 RTGCPHYS GCPhys;
12590 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12591 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12592 AssertRCReturn(rc, rc);
12593
12594 VBOXSTRICTRC rcStrict;
12595 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12596 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
12597 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12598 if (!pExitRec)
12599 {
12600 /*
12601 * If we succeed, resume guest execution.
12602 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12603 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12604 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12605 * weird case. See @bugref{6043}.
12606 */
12607 PVM pVM = pVCpu->CTX_SUFF(pVM);
12608 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12609 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12610 if ( rcStrict == VINF_SUCCESS
12611 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12612 || rcStrict == VERR_PAGE_NOT_PRESENT)
12613 {
12614 /* Successfully handled MMIO operation. */
12615 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12616 | HM_CHANGED_GUEST_RSP
12617 | HM_CHANGED_GUEST_RFLAGS
12618 | HM_CHANGED_GUEST_APIC_TPR);
12619 rcStrict = VINF_SUCCESS;
12620 }
12621 }
12622 else
12623 {
12624 /*
12625 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12626 */
12627 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
12628 int rc2 = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12629 AssertRCReturn(rc2, rc2);
12630
12631 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12632 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12633
12634 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12635 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12636
12637 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12638 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12639 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12640 }
12641 return VBOXSTRICTRC_TODO(rcStrict);
12642}
12643
12644
12645/**
12646 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12647 * VM-exit.
12648 */
12649HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12650{
12651 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12652 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12653
12654 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12655 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12656 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12657 {
12658 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12659 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12660 Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12661 }
12662 else
12663 {
12664 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12665 rcStrict1 = VINF_SUCCESS;
12666 return rcStrict1;
12667 }
12668
12669 RTGCPHYS GCPhys;
12670 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12671 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12672 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12673 AssertRCReturn(rc, rc);
12674
12675 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12676 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12677
12678 RTGCUINT uErrorCode = 0;
12679 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12680 uErrorCode |= X86_TRAP_PF_ID;
12681 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12682 uErrorCode |= X86_TRAP_PF_RW;
12683 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12684 uErrorCode |= X86_TRAP_PF_P;
12685
12686 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12687
12688 Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12689 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12690
12691 /* Handle the pagefault trap for the nested shadow table. */
12692 PVM pVM = pVCpu->CTX_SUFF(pVM);
12693 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12694 TRPMResetTrap(pVCpu);
12695
12696 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12697 if ( rcStrict2 == VINF_SUCCESS
12698 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12699 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12700 {
12701 /* Successfully synced our nested page tables. */
12702 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12703 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12704 | HM_CHANGED_GUEST_RSP
12705 | HM_CHANGED_GUEST_RFLAGS);
12706 return VINF_SUCCESS;
12707 }
12708
12709 Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12710 return rcStrict2;
12711}
12712
12713/** @} */
12714
12715/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12716/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
12717/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12718
12719/** @name VM-exit exception handlers.
12720 * @{
12721 */
12722
12723/**
12724 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12725 */
12726static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12727{
12728 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12729 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12730
12731 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
12732 AssertRCReturn(rc, rc);
12733
12734 if (!(pMixedCtx->cr0 & X86_CR0_NE))
12735 {
12736 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12737 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12738
12739 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12740 * provides VM-exit instruction length. If this causes problem later,
12741 * disassemble the instruction like it's done on AMD-V. */
12742 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12743 AssertRCReturn(rc2, rc2);
12744 return rc;
12745 }
12746
12747 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12748 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12749 return rc;
12750}
12751
12752
12753/**
12754 * VM-exit exception handler for \#BP (Breakpoint exception).
12755 */
12756static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12757{
12758 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12759 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12760
12761 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12762 AssertRCReturn(rc, rc);
12763
12764 rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx));
12765 if (rc == VINF_EM_RAW_GUEST_TRAP)
12766 {
12767 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12768 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12769 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12770 AssertRCReturn(rc, rc);
12771
12772 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12773 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12774 }
12775
12776 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
12777 return rc;
12778}
12779
12780
12781/**
12782 * VM-exit exception handler for \#AC (alignment check exception).
12783 */
12784static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12785{
12786 RT_NOREF_PV(pMixedCtx);
12787 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12788
12789 /*
12790 * Re-inject it. We'll detect any nesting before getting here.
12791 */
12792 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12793 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12794 AssertRCReturn(rc, rc);
12795 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
12796
12797 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12798 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12799 return VINF_SUCCESS;
12800}
12801
12802
12803/**
12804 * VM-exit exception handler for \#DB (Debug exception).
12805 */
12806static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12807{
12808 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12809 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
12810
12811 /*
12812 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
12813 * for processing.
12814 */
12815 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12816
12817 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
12818 uint64_t uDR6 = X86_DR6_INIT_VAL;
12819 uDR6 |= ( pVmxTransient->uExitQualification
12820 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
12821
12822 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
12823 Log6Func(("rc=%Rrc\n", rc));
12824 if (rc == VINF_EM_RAW_GUEST_TRAP)
12825 {
12826 /*
12827 * The exception was for the guest. Update DR6, DR7.GD and
12828 * IA32_DEBUGCTL.LBR before forwarding it.
12829 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
12830 */
12831 VMMRZCallRing3Disable(pVCpu);
12832 HM_DISABLE_PREEMPT();
12833
12834 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
12835 pMixedCtx->dr[6] |= uDR6;
12836 if (CPUMIsGuestDebugStateActive(pVCpu))
12837 ASMSetDR6(pMixedCtx->dr[6]);
12838
12839 HM_RESTORE_PREEMPT();
12840 VMMRZCallRing3Enable(pVCpu);
12841
12842 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12843 AssertRCReturn(rc, rc);
12844
12845 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
12846 pMixedCtx->dr[7] &= ~X86_DR7_GD;
12847
12848 /* Paranoia. */
12849 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
12850 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
12851
12852 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
12853 AssertRCReturn(rc, rc);
12854
12855 /*
12856 * Raise #DB in the guest.
12857 *
12858 * It is important to reflect exactly what the VM-exit gave us (preserving the
12859 * interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
12860 * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
12861 * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
12862 *
12863 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
12864 * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
12865 */
12866 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12867 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12868 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12869 AssertRCReturn(rc, rc);
12870 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12871 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12872 return VINF_SUCCESS;
12873 }
12874
12875 /*
12876 * Not a guest trap, must be a hypervisor related debug event then.
12877 * Update DR6 in case someone is interested in it.
12878 */
12879 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
12880 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
12881 CPUMSetHyperDR6(pVCpu, uDR6);
12882
12883 return rc;
12884}
12885
12886/**
12887 * VM-exit exception handler for \#GP (General-protection exception).
12888 *
12889 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
12890 */
12891static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12892{
12893 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12894 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
12895
12896 int rc;
12897 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
12898 { /* likely */ }
12899 else
12900 {
12901#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12902 Assert(pVCpu->hm.s.fUsingDebugLoop);
12903#endif
12904 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
12905 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12906 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12907 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12908 rc |= hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12909 AssertRCReturn(rc, rc);
12910 Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
12911 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
12912 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12913 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12914 return rc;
12915 }
12916
12917 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
12918 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
12919
12920 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
12921 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12922 AssertRCReturn(rc, rc);
12923
12924 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12925 uint32_t cbOp = 0;
12926 PVM pVM = pVCpu->CTX_SUFF(pVM);
12927 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12928 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
12929 if (RT_SUCCESS(rc))
12930 {
12931 rc = VINF_SUCCESS;
12932 Assert(cbOp == pDis->cbInstr);
12933 Log4Func(("Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12934 switch (pDis->pCurInstr->uOpcode)
12935 {
12936 case OP_CLI:
12937 {
12938 pMixedCtx->eflags.Bits.u1IF = 0;
12939 pMixedCtx->eflags.Bits.u1RF = 0;
12940 pMixedCtx->rip += pDis->cbInstr;
12941 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12942 if ( !fDbgStepping
12943 && pMixedCtx->eflags.Bits.u1TF)
12944 {
12945 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
12946 AssertRCReturn(rc, rc);
12947 }
12948 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
12949 break;
12950 }
12951
12952 case OP_STI:
12953 {
12954 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
12955 pMixedCtx->eflags.Bits.u1IF = 1;
12956 pMixedCtx->eflags.Bits.u1RF = 0;
12957 pMixedCtx->rip += pDis->cbInstr;
12958 if (!fOldIF)
12959 {
12960 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
12961 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
12962 }
12963 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12964 if ( !fDbgStepping
12965 && pMixedCtx->eflags.Bits.u1TF)
12966 {
12967 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
12968 AssertRCReturn(rc, rc);
12969 }
12970 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
12971 break;
12972 }
12973
12974 case OP_HLT:
12975 {
12976 rc = VINF_EM_HALT;
12977 pMixedCtx->rip += pDis->cbInstr;
12978 pMixedCtx->eflags.Bits.u1RF = 0;
12979 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12980 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
12981 break;
12982 }
12983
12984 case OP_POPF:
12985 {
12986 Log4Func(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12987 uint32_t cbParm;
12988 uint32_t uMask;
12989 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12990 if (pDis->fPrefix & DISPREFIX_OPSIZE)
12991 {
12992 cbParm = 4;
12993 uMask = 0xffffffff;
12994 }
12995 else
12996 {
12997 cbParm = 2;
12998 uMask = 0xffff;
12999 }
13000
13001 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13002 RTGCPTR GCPtrStack = 0;
13003 X86EFLAGS Eflags;
13004 Eflags.u32 = 0;
13005 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13006 &GCPtrStack);
13007 if (RT_SUCCESS(rc))
13008 {
13009 Assert(sizeof(Eflags.u32) >= cbParm);
13010 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13011 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13012 }
13013 if (RT_FAILURE(rc))
13014 {
13015 rc = VERR_EM_INTERPRETER;
13016 break;
13017 }
13018 Log4Func(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13019 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13020 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13021 pMixedCtx->esp += cbParm;
13022 pMixedCtx->esp &= uMask;
13023 pMixedCtx->rip += pDis->cbInstr;
13024 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13025 | HM_CHANGED_GUEST_RSP
13026 | HM_CHANGED_GUEST_RFLAGS);
13027 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13028 POPF restores EFLAGS.TF. */
13029 if ( !fDbgStepping
13030 && fGstStepping)
13031 {
13032 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13033 AssertRCReturn(rc, rc);
13034 }
13035 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13036 break;
13037 }
13038
13039 case OP_PUSHF:
13040 {
13041 uint32_t cbParm;
13042 uint32_t uMask;
13043 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13044 {
13045 cbParm = 4;
13046 uMask = 0xffffffff;
13047 }
13048 else
13049 {
13050 cbParm = 2;
13051 uMask = 0xffff;
13052 }
13053
13054 /* Get the stack pointer & push the contents of eflags onto the stack. */
13055 RTGCPTR GCPtrStack = 0;
13056 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13057 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13058 if (RT_FAILURE(rc))
13059 {
13060 rc = VERR_EM_INTERPRETER;
13061 break;
13062 }
13063 X86EFLAGS Eflags = pMixedCtx->eflags;
13064 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13065 Eflags.Bits.u1RF = 0;
13066 Eflags.Bits.u1VM = 0;
13067
13068 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13069 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13070 {
13071 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13072 rc = VERR_EM_INTERPRETER;
13073 break;
13074 }
13075 Log4Func(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13076 pMixedCtx->esp -= cbParm;
13077 pMixedCtx->esp &= uMask;
13078 pMixedCtx->rip += pDis->cbInstr;
13079 pMixedCtx->eflags.Bits.u1RF = 0;
13080 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13081 | HM_CHANGED_GUEST_RSP
13082 | HM_CHANGED_GUEST_RFLAGS);
13083 if ( !fDbgStepping
13084 && pMixedCtx->eflags.Bits.u1TF)
13085 {
13086 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13087 AssertRCReturn(rc, rc);
13088 }
13089 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13090 break;
13091 }
13092
13093 case OP_IRET:
13094 {
13095 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13096 * instruction reference. */
13097 RTGCPTR GCPtrStack = 0;
13098 uint32_t uMask = 0xffff;
13099 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13100 uint16_t aIretFrame[3];
13101 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13102 {
13103 rc = VERR_EM_INTERPRETER;
13104 break;
13105 }
13106 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13107 &GCPtrStack);
13108 if (RT_SUCCESS(rc))
13109 {
13110 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13111 PGMACCESSORIGIN_HM));
13112 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13113 }
13114 if (RT_FAILURE(rc))
13115 {
13116 rc = VERR_EM_INTERPRETER;
13117 break;
13118 }
13119 pMixedCtx->eip = 0;
13120 pMixedCtx->ip = aIretFrame[0];
13121 pMixedCtx->cs.Sel = aIretFrame[1];
13122 pMixedCtx->cs.ValidSel = aIretFrame[1];
13123 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13124 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13125 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13126 pMixedCtx->sp += sizeof(aIretFrame);
13127 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13128 | HM_CHANGED_GUEST_CS
13129 | HM_CHANGED_GUEST_RSP
13130 | HM_CHANGED_GUEST_RFLAGS);
13131 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13132 if ( !fDbgStepping
13133 && fGstStepping)
13134 {
13135 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13136 AssertRCReturn(rc, rc);
13137 }
13138 Log4Func(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13139 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13140 break;
13141 }
13142
13143 case OP_INT:
13144 {
13145 uint16_t uVector = pDis->Param1.uValue & 0xff;
13146 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13147 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13148 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13149 break;
13150 }
13151
13152 case OP_INTO:
13153 {
13154 if (pMixedCtx->eflags.Bits.u1OF)
13155 {
13156 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13157 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13158 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13159 }
13160 else
13161 {
13162 pMixedCtx->eflags.Bits.u1RF = 0;
13163 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
13164 }
13165 break;
13166 }
13167
13168 default:
13169 {
13170 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13171 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13172 EMCODETYPE_SUPERVISOR);
13173 rc = VBOXSTRICTRC_VAL(rc2);
13174 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13175 /** @todo We have to set pending-debug exceptions here when the guest is
13176 * single-stepping depending on the instruction that was interpreted. */
13177 Log4Func(("#GP rc=%Rrc\n", rc));
13178 break;
13179 }
13180 }
13181 }
13182 else
13183 rc = VERR_EM_INTERPRETER;
13184
13185 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13186 ("#GP Unexpected rc=%Rrc\n", rc));
13187 return rc;
13188}
13189
13190
13191/**
13192 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13193 * the exception reported in the VMX transient structure back into the VM.
13194 *
13195 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13196 * up-to-date.
13197 */
13198static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13199{
13200 RT_NOREF_PV(pMixedCtx);
13201 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13202#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13203 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13204 ("uVector=%#x u32XcptBitmap=%#X32\n",
13205 VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13206#endif
13207
13208 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13209 hmR0VmxCheckExitDueToEventDelivery(). */
13210 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13211 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13212 AssertRCReturn(rc, rc);
13213 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13214
13215#ifdef DEBUG_ramshankar
13216 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS
13217 | CPUMCTX_EXTRN_RIP);
13218 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13219 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13220#endif
13221
13222 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13223 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13224 return VINF_SUCCESS;
13225}
13226
13227
13228/**
13229 * VM-exit exception handler for \#PF (Page-fault exception).
13230 */
13231static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13232{
13233 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13234 PVM pVM = pVCpu->CTX_SUFF(pVM);
13235 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13236 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13237 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13238 AssertRCReturn(rc, rc);
13239
13240 if (!pVM->hm.s.fNestedPaging)
13241 { /* likely */ }
13242 else
13243 {
13244#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13245 Assert(pVCpu->hm.s.fUsingDebugLoop);
13246#endif
13247 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13248 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13249 {
13250 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13251 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13252 }
13253 else
13254 {
13255 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13256 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13257 Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13258 }
13259 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13260 return rc;
13261 }
13262
13263 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13264 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13265 if (pVmxTransient->fVectoringPF)
13266 {
13267 Assert(pVCpu->hm.s.Event.fPending);
13268 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13269 }
13270
13271 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13272 AssertRCReturn(rc, rc);
13273
13274 Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13275 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13276
13277 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13278 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13279 (RTGCPTR)pVmxTransient->uExitQualification);
13280
13281 Log4Func(("#PF: rc=%Rrc\n", rc));
13282 if (rc == VINF_SUCCESS)
13283 {
13284 /*
13285 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13286 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13287 */
13288 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13289 TRPMResetTrap(pVCpu);
13290 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13291 return rc;
13292 }
13293
13294 if (rc == VINF_EM_RAW_GUEST_TRAP)
13295 {
13296 if (!pVmxTransient->fVectoringDoublePF)
13297 {
13298 /* It's a guest page fault and needs to be reflected to the guest. */
13299 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13300 TRPMResetTrap(pVCpu);
13301 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13302 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13303 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13304 }
13305 else
13306 {
13307 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13308 TRPMResetTrap(pVCpu);
13309 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13310 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13311 Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13312 }
13313
13314 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13315 return VINF_SUCCESS;
13316 }
13317
13318 TRPMResetTrap(pVCpu);
13319 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13320 return rc;
13321}
13322
13323/** @} */
13324
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette