VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 72784

Last change on this file since 72784 was 72784, checked in by vboxsync, 6 years ago

VMM/HMVMXR0: Update ProcCtls when changed.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 572.7 KB
Line 
1/* $Id: HMVMXR0.cpp 72784 2018-07-02 06:40:19Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/x86.h>
25#include <iprt/asm-amd64-x86.h>
26#include <iprt/thread.h>
27
28#include <VBox/vmm/pdmapi.h>
29#include <VBox/vmm/dbgf.h>
30#include <VBox/vmm/iem.h>
31#include <VBox/vmm/iom.h>
32#include <VBox/vmm/selm.h>
33#include <VBox/vmm/tm.h>
34#include <VBox/vmm/gim.h>
35#include <VBox/vmm/apic.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#include "HMInternal.h"
40#include <VBox/vmm/vm.h>
41#include "HMVMXR0.h"
42#include "dtrace/VBoxVMM.h"
43
44#ifdef DEBUG_ramshankar
45# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50# define HMVMX_ALWAYS_TRAP_PF
51# define HMVMX_ALWAYS_FLUSH_TLB
52# define HMVMX_ALWAYS_SWAP_EFER
53#endif
54
55
56/*********************************************************************************************************************************
57* Defined Constants And Macros *
58*********************************************************************************************************************************/
59/** Use the function table. */
60#define HMVMX_USE_FUNCTION_TABLE
61
62/** Determine which tagged-TLB flush handler to use. */
63#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68/** @name HMVMX_READ_XXX
69 * Flags to skip redundant reads of some common VMCS fields that are not part of
70 * the guest-CPU or VCPU state but are needed while handling VM-exits.
71 */
72#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
73#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
74#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
75#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
76#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
77#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
78#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
79/** @} */
80
81/**
82 * States of the VMCS.
83 *
84 * This does not reflect all possible VMCS states but currently only those
85 * needed for maintaining the VMCS consistently even when thread-context hooks
86 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
87 */
88#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
89#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
90#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
91
92/**
93 * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
94 * guest using hardware-assisted VMX.
95 *
96 * This excludes state like GPRs (other than RSP) which are always are
97 * swapped and restored across the world-switch and also registers like EFER,
98 * MSR which cannot be modified by the guest without causing a VM-exit.
99 */
100#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
101 | CPUMCTX_EXTRN_RFLAGS \
102 | CPUMCTX_EXTRN_RSP \
103 | CPUMCTX_EXTRN_SREG_MASK \
104 | CPUMCTX_EXTRN_TABLE_MASK \
105 | CPUMCTX_EXTRN_KERNEL_GS_BASE \
106 | CPUMCTX_EXTRN_SYSCALL_MSRS \
107 | CPUMCTX_EXTRN_SYSENTER_MSRS \
108 | CPUMCTX_EXTRN_TSC_AUX \
109 | CPUMCTX_EXTRN_OTHER_MSRS \
110 | CPUMCTX_EXTRN_CR0 \
111 | CPUMCTX_EXTRN_CR3 \
112 | CPUMCTX_EXTRN_CR4 \
113 | CPUMCTX_EXTRN_DR7 \
114 | CPUMCTX_EXTRN_HM_VMX_MASK)
115
116/**
117 * Exception bitmap mask for real-mode guests (real-on-v86).
118 *
119 * We need to intercept all exceptions manually except:
120 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
121 * due to bugs in Intel CPUs.
122 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
123 * support.
124 */
125#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
126 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
127 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
128 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
129 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
130 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
131 | RT_BIT(X86_XCPT_XF))
132
133/**
134 * Exception bitmap mask for all contributory exceptions.
135 *
136 * Page fault is deliberately excluded here as it's conditional as to whether
137 * it's contributory or benign. Page faults are handled separately.
138 */
139#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
140 | RT_BIT(X86_XCPT_DE))
141
142/** Maximum VM-instruction error number. */
143#define HMVMX_INSTR_ERROR_MAX 28
144
145/** Profiling macro. */
146#ifdef HM_PROFILE_EXIT_DISPATCH
147# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
148# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
149#else
150# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
151# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
152#endif
153
154/** Assert that preemption is disabled or covered by thread-context hooks. */
155#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
156 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
157
158/** Assert that we haven't migrated CPUs when thread-context hooks are not
159 * used. */
160#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
161 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
162 ("Illegal migration! Entered on CPU %u Current %u\n", \
163 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
164
165/** Helper macro for VM-exit handlers called unexpectedly. */
166#define HMVMX_RETURN_UNEXPECTED_EXIT() \
167 do { \
168 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
169 return VERR_VMX_UNEXPECTED_EXIT; \
170 } while (0)
171
172/** Macro for saving segment registers from VMCS into the guest-CPU
173 * context. */
174#ifdef VMX_USE_CACHED_VMCS_ACCESSES
175# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
176 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
177 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
178#else
179# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
180 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
181 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
182#endif
183
184
185/*********************************************************************************************************************************
186* Structures and Typedefs *
187*********************************************************************************************************************************/
188/**
189 * VMX transient state.
190 *
191 * A state structure for holding miscellaneous information across
192 * VMX non-root operation and restored after the transition.
193 */
194typedef struct VMXTRANSIENT
195{
196 /** The host's rflags/eflags. */
197 RTCCUINTREG fEFlags;
198#if HC_ARCH_BITS == 32
199 uint32_t u32Alignment0;
200#endif
201 /** The guest's TPR value used for TPR shadowing. */
202 uint8_t u8GuestTpr;
203 /** Alignment. */
204 uint8_t abAlignment0[7];
205
206 /** The basic VM-exit reason. */
207 uint16_t uExitReason;
208 /** Alignment. */
209 uint16_t u16Alignment0;
210 /** The VM-exit interruption error code. */
211 uint32_t uExitIntErrorCode;
212 /** The VM-exit exit code qualification. */
213 uint64_t uExitQualification;
214
215 /** The VM-exit interruption-information field. */
216 uint32_t uExitIntInfo;
217 /** The VM-exit instruction-length field. */
218 uint32_t cbInstr;
219 /** The VM-exit instruction-information field. */
220 union
221 {
222 /** Plain unsigned int representation. */
223 uint32_t u;
224 /** INS and OUTS information. */
225 struct
226 {
227 uint32_t u7Reserved0 : 7;
228 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
229 uint32_t u3AddrSize : 3;
230 uint32_t u5Reserved1 : 5;
231 /** The segment register (X86_SREG_XXX). */
232 uint32_t iSegReg : 3;
233 uint32_t uReserved2 : 14;
234 } StrIo;
235 /** INVEPT, INVVPID, INVPCID information. */
236 struct
237 {
238 /** Scaling; 0=no scaling, 1=scale-by-2, 2=scale-by-4, 3=scale-by-8. */
239 uint32_t u2Scaling : 2;
240 uint32_t u5Reserved0 : 5;
241 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
242 uint32_t u3AddrSize : 3;
243 uint32_t u1Reserved0 : 1;
244 uint32_t u4Reserved0 : 4;
245 /** The segment register (X86_SREG_XXX). */
246 uint32_t iSegReg : 3;
247 /** The index register (X86_GREG_XXX). */
248 uint32_t iIdxReg : 4;
249 /** Set if index register is invalid. */
250 uint32_t fIdxRegValid : 1;
251 /** The base register (X86_GREG_XXX). */
252 uint32_t iBaseReg : 4;
253 /** Set if base register is invalid. */
254 uint32_t fBaseRegValid : 1;
255 /** Register 2 (X86_GREG_XXX). */
256 uint32_t iReg2 : 4;
257 } Inv;
258 } ExitInstrInfo;
259 /** Whether the VM-entry failed or not. */
260 bool fVMEntryFailed;
261 /** Alignment. */
262 uint8_t abAlignment1[3];
263
264 /** The VM-entry interruption-information field. */
265 uint32_t uEntryIntInfo;
266 /** The VM-entry exception error code field. */
267 uint32_t uEntryXcptErrorCode;
268 /** The VM-entry instruction length field. */
269 uint32_t cbEntryInstr;
270
271 /** IDT-vectoring information field. */
272 uint32_t uIdtVectoringInfo;
273 /** IDT-vectoring error code. */
274 uint32_t uIdtVectoringErrorCode;
275
276 /** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
277 uint32_t fVmcsFieldsRead;
278
279 /** Whether the guest debug state was active at the time of VM-exit. */
280 bool fWasGuestDebugStateActive;
281 /** Whether the hyper debug state was active at the time of VM-exit. */
282 bool fWasHyperDebugStateActive;
283 /** Whether TSC-offsetting should be setup before VM-entry. */
284 bool fUpdateTscOffsettingAndPreemptTimer;
285 /** Whether the VM-exit was caused by a page-fault during delivery of a
286 * contributory exception or a page-fault. */
287 bool fVectoringDoublePF;
288 /** Whether the VM-exit was caused by a page-fault during delivery of an
289 * external interrupt or NMI. */
290 bool fVectoringPF;
291} VMXTRANSIENT;
292AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
293AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
294AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
295AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
296AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
297/** Pointer to VMX transient state. */
298typedef VMXTRANSIENT *PVMXTRANSIENT;
299
300
301/**
302 * MSR-bitmap read permissions.
303 */
304typedef enum VMXMSREXITREAD
305{
306 /** Reading this MSR causes a VM-exit. */
307 VMXMSREXIT_INTERCEPT_READ = 0xb,
308 /** Reading this MSR does not cause a VM-exit. */
309 VMXMSREXIT_PASSTHRU_READ
310} VMXMSREXITREAD;
311/** Pointer to MSR-bitmap read permissions. */
312typedef VMXMSREXITREAD* PVMXMSREXITREAD;
313
314/**
315 * MSR-bitmap write permissions.
316 */
317typedef enum VMXMSREXITWRITE
318{
319 /** Writing to this MSR causes a VM-exit. */
320 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
321 /** Writing to this MSR does not cause a VM-exit. */
322 VMXMSREXIT_PASSTHRU_WRITE
323} VMXMSREXITWRITE;
324/** Pointer to MSR-bitmap write permissions. */
325typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
326
327
328/**
329 * VMX VM-exit handler.
330 *
331 * @returns Strict VBox status code (i.e. informational status codes too).
332 * @param pVCpu The cross context virtual CPU structure.
333 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
334 * out-of-sync. Make sure to update the required
335 * fields before using them.
336 * @param pVmxTransient Pointer to the VMX-transient structure.
337 */
338#ifndef HMVMX_USE_FUNCTION_TABLE
339typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
340#else
341typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
342/** Pointer to VM-exit handler. */
343typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
344#endif
345
346/**
347 * VMX VM-exit handler, non-strict status code.
348 *
349 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
350 *
351 * @returns VBox status code, no informational status code returned.
352 * @param pVCpu The cross context virtual CPU structure.
353 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
354 * out-of-sync. Make sure to update the required
355 * fields before using them.
356 * @param pVmxTransient Pointer to the VMX-transient structure.
357 *
358 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
359 * use of that status code will be replaced with VINF_EM_SOMETHING
360 * later when switching over to IEM.
361 */
362#ifndef HMVMX_USE_FUNCTION_TABLE
363typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
364#else
365typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
366#endif
367
368
369/*********************************************************************************************************************************
370* Internal Functions *
371*********************************************************************************************************************************/
372static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
373static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
374static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
375static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
376static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
377 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
378#if HC_ARCH_BITS == 32
379static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
380#endif
381#ifndef HMVMX_USE_FUNCTION_TABLE
382DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
383# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
384# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
385#else
386# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
387# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
388#endif
389
390
391/** @name VM-exit handlers.
392 * @{
393 */
394static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
395static FNVMXEXITHANDLER hmR0VmxExitExtInt;
396static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
397static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
398static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
399static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
400static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
401static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
402static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
403static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
404static FNVMXEXITHANDLER hmR0VmxExitCpuid;
405static FNVMXEXITHANDLER hmR0VmxExitGetsec;
406static FNVMXEXITHANDLER hmR0VmxExitHlt;
407static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
408static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
409static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
410static FNVMXEXITHANDLER hmR0VmxExitVmcall;
411static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
412static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
413static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
414static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
415static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
416static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
417static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
418static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
419static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
420static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
421static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
422static FNVMXEXITHANDLER hmR0VmxExitMwait;
423static FNVMXEXITHANDLER hmR0VmxExitMtf;
424static FNVMXEXITHANDLER hmR0VmxExitMonitor;
425static FNVMXEXITHANDLER hmR0VmxExitPause;
426static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
427static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
428static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
429static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
430static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
431static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
432static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
433static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
434static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
435static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
436static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
437static FNVMXEXITHANDLER hmR0VmxExitRdrand;
438static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
439/** @} */
440
441static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
442static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
443static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
444static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
445static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
446static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
447static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
448static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
449
450
451/*********************************************************************************************************************************
452* Global Variables *
453*********************************************************************************************************************************/
454#ifdef HMVMX_USE_FUNCTION_TABLE
455
456/**
457 * VMX_EXIT dispatch table.
458 */
459static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
460{
461 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
462 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
463 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
464 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
465 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
466 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
467 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
468 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
469 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
470 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
471 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
472 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
473 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
474 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
475 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
476 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
477 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
478 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
479 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
480 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
481 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
482 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
483 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
484 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
485 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
486 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
487 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
488 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
489 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
490 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
491 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
492 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
493 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
494 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
495 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
496 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
497 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
498 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
499 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
500 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
501 /* 40 UNDEFINED */ hmR0VmxExitPause,
502 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
503 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
504 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
505 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
506 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
507 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
508 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
509 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
510 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
511 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
512 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
513 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
514 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
515 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
516 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
517 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
518 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
519 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
520 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
521 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
522 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
523 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
524 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
525 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
526};
527#endif /* HMVMX_USE_FUNCTION_TABLE */
528
529#ifdef VBOX_STRICT
530static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
531{
532 /* 0 */ "(Not Used)",
533 /* 1 */ "VMCALL executed in VMX root operation.",
534 /* 2 */ "VMCLEAR with invalid physical address.",
535 /* 3 */ "VMCLEAR with VMXON pointer.",
536 /* 4 */ "VMLAUNCH with non-clear VMCS.",
537 /* 5 */ "VMRESUME with non-launched VMCS.",
538 /* 6 */ "VMRESUME after VMXOFF",
539 /* 7 */ "VM-entry with invalid control fields.",
540 /* 8 */ "VM-entry with invalid host state fields.",
541 /* 9 */ "VMPTRLD with invalid physical address.",
542 /* 10 */ "VMPTRLD with VMXON pointer.",
543 /* 11 */ "VMPTRLD with incorrect revision identifier.",
544 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
545 /* 13 */ "VMWRITE to read-only VMCS component.",
546 /* 14 */ "(Not Used)",
547 /* 15 */ "VMXON executed in VMX root operation.",
548 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
549 /* 17 */ "VM-entry with non-launched executing VMCS.",
550 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
551 /* 19 */ "VMCALL with non-clear VMCS.",
552 /* 20 */ "VMCALL with invalid VM-exit control fields.",
553 /* 21 */ "(Not Used)",
554 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
555 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
556 /* 24 */ "VMCALL with invalid SMM-monitor features.",
557 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
558 /* 26 */ "VM-entry with events blocked by MOV SS.",
559 /* 27 */ "(Not Used)",
560 /* 28 */ "Invalid operand to INVEPT/INVVPID."
561};
562#endif /* VBOX_STRICT */
563
564
565
566/**
567 * Updates the VM's last error record.
568 *
569 * If there was a VMX instruction error, reads the error data from the VMCS and
570 * updates VCPU's last error record as well.
571 *
572 * @param pVM The cross context VM structure.
573 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
574 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
575 * VERR_VMX_INVALID_VMCS_FIELD.
576 * @param rc The error code.
577 */
578static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
579{
580 AssertPtr(pVM);
581 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
582 || rc == VERR_VMX_UNABLE_TO_START_VM)
583 {
584 AssertPtrReturnVoid(pVCpu);
585 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
586 }
587 pVM->hm.s.lLastError = rc;
588}
589
590
591/**
592 * Reads the VM-entry interruption-information field from the VMCS into the VMX
593 * transient structure.
594 *
595 * @returns VBox status code.
596 * @param pVmxTransient Pointer to the VMX transient structure.
597 *
598 * @remarks No-long-jump zone!!!
599 */
600DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
601{
602 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
603 AssertRCReturn(rc, rc);
604 return VINF_SUCCESS;
605}
606
607#ifdef VBOX_STRICT
608/**
609 * Reads the VM-entry exception error code field from the VMCS into
610 * the VMX transient structure.
611 *
612 * @returns VBox status code.
613 * @param pVmxTransient Pointer to the VMX transient structure.
614 *
615 * @remarks No-long-jump zone!!!
616 */
617DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
618{
619 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
620 AssertRCReturn(rc, rc);
621 return VINF_SUCCESS;
622}
623
624
625/**
626 * Reads the VM-entry exception error code field from the VMCS into
627 * the VMX transient structure.
628 *
629 * @returns VBox status code.
630 * @param pVmxTransient Pointer to the VMX transient structure.
631 *
632 * @remarks No-long-jump zone!!!
633 */
634DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
635{
636 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
637 AssertRCReturn(rc, rc);
638 return VINF_SUCCESS;
639}
640#endif /* VBOX_STRICT */
641
642
643/**
644 * Reads the VM-exit interruption-information field from the VMCS into the VMX
645 * transient structure.
646 *
647 * @returns VBox status code.
648 * @param pVmxTransient Pointer to the VMX transient structure.
649 */
650DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
651{
652 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
653 {
654 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
655 AssertRCReturn(rc,rc);
656 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_INFO;
657 }
658 return VINF_SUCCESS;
659}
660
661
662/**
663 * Reads the VM-exit interruption error code from the VMCS into the VMX
664 * transient structure.
665 *
666 * @returns VBox status code.
667 * @param pVmxTransient Pointer to the VMX transient structure.
668 */
669DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
670{
671 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
672 {
673 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
674 AssertRCReturn(rc, rc);
675 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
676 }
677 return VINF_SUCCESS;
678}
679
680
681/**
682 * Reads the VM-exit instruction length field from the VMCS into the VMX
683 * transient structure.
684 *
685 * @returns VBox status code.
686 * @param pVmxTransient Pointer to the VMX transient structure.
687 */
688DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
689{
690 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
691 {
692 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
693 AssertRCReturn(rc, rc);
694 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_LEN;
695 }
696 return VINF_SUCCESS;
697}
698
699
700/**
701 * Reads the VM-exit instruction-information field from the VMCS into
702 * the VMX transient structure.
703 *
704 * @returns VBox status code.
705 * @param pVmxTransient Pointer to the VMX transient structure.
706 */
707DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
708{
709 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
710 {
711 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
712 AssertRCReturn(rc, rc);
713 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_INFO;
714 }
715 return VINF_SUCCESS;
716}
717
718
719/**
720 * Reads the exit code qualification from the VMCS into the VMX transient
721 * structure.
722 *
723 * @returns VBox status code.
724 * @param pVCpu The cross context virtual CPU structure of the
725 * calling EMT. (Required for the VMCS cache case.)
726 * @param pVmxTransient Pointer to the VMX transient structure.
727 */
728DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
729{
730 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
731 {
732 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
733 AssertRCReturn(rc, rc);
734 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION;
735 }
736 return VINF_SUCCESS;
737}
738
739
740/**
741 * Reads the IDT-vectoring information field from the VMCS into the VMX
742 * transient structure.
743 *
744 * @returns VBox status code.
745 * @param pVmxTransient Pointer to the VMX transient structure.
746 *
747 * @remarks No-long-jump zone!!!
748 */
749DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
750{
751 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
752 {
753 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
754 AssertRCReturn(rc, rc);
755 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_INFO;
756 }
757 return VINF_SUCCESS;
758}
759
760
761/**
762 * Reads the IDT-vectoring error code from the VMCS into the VMX
763 * transient structure.
764 *
765 * @returns VBox status code.
766 * @param pVmxTransient Pointer to the VMX transient structure.
767 */
768DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
769{
770 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
771 {
772 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
773 AssertRCReturn(rc, rc);
774 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
775 }
776 return VINF_SUCCESS;
777}
778
779
780/**
781 * Enters VMX root mode operation on the current CPU.
782 *
783 * @returns VBox status code.
784 * @param pVM The cross context VM structure. Can be
785 * NULL, after a resume.
786 * @param HCPhysCpuPage Physical address of the VMXON region.
787 * @param pvCpuPage Pointer to the VMXON region.
788 */
789static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
790{
791 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
792 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
793 Assert(pvCpuPage);
794 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
795
796 if (pVM)
797 {
798 /* Write the VMCS revision dword to the VMXON region. */
799 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
800 }
801
802 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
803 RTCCUINTREG fEFlags = ASMIntDisableFlags();
804
805 /* Enable the VMX bit in CR4 if necessary. */
806 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
807
808 /* Enter VMX root mode. */
809 int rc = VMXEnable(HCPhysCpuPage);
810 if (RT_FAILURE(rc))
811 {
812 if (!(uOldCr4 & X86_CR4_VMXE))
813 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
814
815 if (pVM)
816 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
817 }
818
819 /* Restore interrupts. */
820 ASMSetFlags(fEFlags);
821 return rc;
822}
823
824
825/**
826 * Exits VMX root mode operation on the current CPU.
827 *
828 * @returns VBox status code.
829 */
830static int hmR0VmxLeaveRootMode(void)
831{
832 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
833
834 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
835 RTCCUINTREG fEFlags = ASMIntDisableFlags();
836
837 /* If we're for some reason not in VMX root mode, then don't leave it. */
838 RTCCUINTREG uHostCR4 = ASMGetCR4();
839
840 int rc;
841 if (uHostCR4 & X86_CR4_VMXE)
842 {
843 /* Exit VMX root mode and clear the VMX bit in CR4. */
844 VMXDisable();
845 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
846 rc = VINF_SUCCESS;
847 }
848 else
849 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
850
851 /* Restore interrupts. */
852 ASMSetFlags(fEFlags);
853 return rc;
854}
855
856
857/**
858 * Allocates and maps one physically contiguous page. The allocated page is
859 * zero'd out. (Used by various VT-x structures).
860 *
861 * @returns IPRT status code.
862 * @param pMemObj Pointer to the ring-0 memory object.
863 * @param ppVirt Where to store the virtual address of the
864 * allocation.
865 * @param pHCPhys Where to store the physical address of the
866 * allocation.
867 */
868static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
869{
870 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
871 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
872 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
873
874 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
875 if (RT_FAILURE(rc))
876 return rc;
877 *ppVirt = RTR0MemObjAddress(*pMemObj);
878 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
879 ASMMemZero32(*ppVirt, PAGE_SIZE);
880 return VINF_SUCCESS;
881}
882
883
884/**
885 * Frees and unmaps an allocated physical page.
886 *
887 * @param pMemObj Pointer to the ring-0 memory object.
888 * @param ppVirt Where to re-initialize the virtual address of
889 * allocation as 0.
890 * @param pHCPhys Where to re-initialize the physical address of the
891 * allocation as 0.
892 */
893static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
894{
895 AssertPtr(pMemObj);
896 AssertPtr(ppVirt);
897 AssertPtr(pHCPhys);
898 if (*pMemObj != NIL_RTR0MEMOBJ)
899 {
900 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
901 AssertRC(rc);
902 *pMemObj = NIL_RTR0MEMOBJ;
903 *ppVirt = 0;
904 *pHCPhys = 0;
905 }
906}
907
908
909/**
910 * Worker function to free VT-x related structures.
911 *
912 * @returns IPRT status code.
913 * @param pVM The cross context VM structure.
914 */
915static void hmR0VmxStructsFree(PVM pVM)
916{
917 for (VMCPUID i = 0; i < pVM->cCpus; i++)
918 {
919 PVMCPU pVCpu = &pVM->aCpus[i];
920 AssertPtr(pVCpu);
921
922 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
923 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
924
925 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
926 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
927
928 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
929 }
930
931 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
932#ifdef VBOX_WITH_CRASHDUMP_MAGIC
933 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
934#endif
935}
936
937
938/**
939 * Worker function to allocate VT-x related VM structures.
940 *
941 * @returns IPRT status code.
942 * @param pVM The cross context VM structure.
943 */
944static int hmR0VmxStructsAlloc(PVM pVM)
945{
946 /*
947 * Initialize members up-front so we can cleanup properly on allocation failure.
948 */
949#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
950 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
951 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
952 pVM->hm.s.vmx.HCPhys##a_Name = 0;
953
954#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
955 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
956 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
957 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
958
959#ifdef VBOX_WITH_CRASHDUMP_MAGIC
960 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
961#endif
962 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
963
964 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
965 for (VMCPUID i = 0; i < pVM->cCpus; i++)
966 {
967 PVMCPU pVCpu = &pVM->aCpus[i];
968 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
969 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
970 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
971 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
972 }
973#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
974#undef VMXLOCAL_INIT_VM_MEMOBJ
975
976 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
977 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
978 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
979 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
980
981 /*
982 * Allocate all the VT-x structures.
983 */
984 int rc = VINF_SUCCESS;
985#ifdef VBOX_WITH_CRASHDUMP_MAGIC
986 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
987 if (RT_FAILURE(rc))
988 goto cleanup;
989 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
990 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
991#endif
992
993 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
994 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
995 {
996 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
997 &pVM->hm.s.vmx.HCPhysApicAccess);
998 if (RT_FAILURE(rc))
999 goto cleanup;
1000 }
1001
1002 /*
1003 * Initialize per-VCPU VT-x structures.
1004 */
1005 for (VMCPUID i = 0; i < pVM->cCpus; i++)
1006 {
1007 PVMCPU pVCpu = &pVM->aCpus[i];
1008 AssertPtr(pVCpu);
1009
1010 /* Allocate the VM control structure (VMCS). */
1011 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1012 if (RT_FAILURE(rc))
1013 goto cleanup;
1014
1015 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1016 if ( PDMHasApic(pVM)
1017 && (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW))
1018 {
1019 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1020 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1021 if (RT_FAILURE(rc))
1022 goto cleanup;
1023 }
1024
1025 /*
1026 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1027 * transparent accesses of specific MSRs.
1028 *
1029 * If the condition for enabling MSR bitmaps changes here, don't forget to
1030 * update HMAreMsrBitmapsAvailable().
1031 */
1032 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1033 {
1034 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1035 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1036 if (RT_FAILURE(rc))
1037 goto cleanup;
1038 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1039 }
1040
1041 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1042 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1043 if (RT_FAILURE(rc))
1044 goto cleanup;
1045
1046 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1047 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1048 if (RT_FAILURE(rc))
1049 goto cleanup;
1050 }
1051
1052 return VINF_SUCCESS;
1053
1054cleanup:
1055 hmR0VmxStructsFree(pVM);
1056 return rc;
1057}
1058
1059
1060/**
1061 * Does global VT-x initialization (called during module initialization).
1062 *
1063 * @returns VBox status code.
1064 */
1065VMMR0DECL(int) VMXR0GlobalInit(void)
1066{
1067#ifdef HMVMX_USE_FUNCTION_TABLE
1068 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1069# ifdef VBOX_STRICT
1070 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1071 Assert(g_apfnVMExitHandlers[i]);
1072# endif
1073#endif
1074 return VINF_SUCCESS;
1075}
1076
1077
1078/**
1079 * Does global VT-x termination (called during module termination).
1080 */
1081VMMR0DECL(void) VMXR0GlobalTerm()
1082{
1083 /* Nothing to do currently. */
1084}
1085
1086
1087/**
1088 * Sets up and activates VT-x on the current CPU.
1089 *
1090 * @returns VBox status code.
1091 * @param pCpu Pointer to the global CPU info struct.
1092 * @param pVM The cross context VM structure. Can be
1093 * NULL after a host resume operation.
1094 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1095 * fEnabledByHost is @c true).
1096 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1097 * @a fEnabledByHost is @c true).
1098 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1099 * enable VT-x on the host.
1100 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1101 */
1102VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1103 void *pvMsrs)
1104{
1105 Assert(pCpu);
1106 Assert(pvMsrs);
1107 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1108
1109 /* Enable VT-x if it's not already enabled by the host. */
1110 if (!fEnabledByHost)
1111 {
1112 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1113 if (RT_FAILURE(rc))
1114 return rc;
1115 }
1116
1117 /*
1118 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1119 * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1120 * invalidated when flushing by VPID.
1121 */
1122 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1123 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1124 {
1125 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1126 pCpu->fFlushAsidBeforeUse = false;
1127 }
1128 else
1129 pCpu->fFlushAsidBeforeUse = true;
1130
1131 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1132 ++pCpu->cTlbFlushes;
1133
1134 return VINF_SUCCESS;
1135}
1136
1137
1138/**
1139 * Deactivates VT-x on the current CPU.
1140 *
1141 * @returns VBox status code.
1142 * @param pCpu Pointer to the global CPU info struct.
1143 * @param pvCpuPage Pointer to the VMXON region.
1144 * @param HCPhysCpuPage Physical address of the VMXON region.
1145 *
1146 * @remarks This function should never be called when SUPR0EnableVTx() or
1147 * similar was used to enable VT-x on the host.
1148 */
1149VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1150{
1151 NOREF(pCpu);
1152 NOREF(pvCpuPage);
1153 NOREF(HCPhysCpuPage);
1154
1155 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1156 return hmR0VmxLeaveRootMode();
1157}
1158
1159
1160/**
1161 * Sets the permission bits for the specified MSR in the MSR bitmap.
1162 *
1163 * @param pVCpu The cross context virtual CPU structure.
1164 * @param uMsr The MSR value.
1165 * @param enmRead Whether reading this MSR causes a VM-exit.
1166 * @param enmWrite Whether writing this MSR causes a VM-exit.
1167 */
1168static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1169{
1170 int32_t iBit;
1171 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1172
1173 /*
1174 * Layout:
1175 * 0x000 - 0x3ff - Low MSR read bits
1176 * 0x400 - 0x7ff - High MSR read bits
1177 * 0x800 - 0xbff - Low MSR write bits
1178 * 0xc00 - 0xfff - High MSR write bits
1179 */
1180 if (uMsr <= 0x00001FFF)
1181 iBit = uMsr;
1182 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1183 {
1184 iBit = uMsr - UINT32_C(0xC0000000);
1185 pbMsrBitmap += 0x400;
1186 }
1187 else
1188 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1189
1190 Assert(iBit <= 0x1fff);
1191 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1192 ASMBitSet(pbMsrBitmap, iBit);
1193 else
1194 ASMBitClear(pbMsrBitmap, iBit);
1195
1196 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1197 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1198 else
1199 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1200}
1201
1202
1203#ifdef VBOX_STRICT
1204/**
1205 * Gets the permission bits for the specified MSR in the MSR bitmap.
1206 *
1207 * @returns VBox status code.
1208 * @retval VINF_SUCCESS if the specified MSR is found.
1209 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1210 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1211 *
1212 * @param pVCpu The cross context virtual CPU structure.
1213 * @param uMsr The MSR.
1214 * @param penmRead Where to store the read permissions.
1215 * @param penmWrite Where to store the write permissions.
1216 */
1217static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1218{
1219 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1220 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1221 int32_t iBit;
1222 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1223
1224 /* See hmR0VmxSetMsrPermission() for the layout. */
1225 if (uMsr <= 0x00001FFF)
1226 iBit = uMsr;
1227 else if ( uMsr >= 0xC0000000
1228 && uMsr <= 0xC0001FFF)
1229 {
1230 iBit = (uMsr - 0xC0000000);
1231 pbMsrBitmap += 0x400;
1232 }
1233 else
1234 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1235
1236 Assert(iBit <= 0x1fff);
1237 if (ASMBitTest(pbMsrBitmap, iBit))
1238 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1239 else
1240 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1241
1242 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1243 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1244 else
1245 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1246 return VINF_SUCCESS;
1247}
1248#endif /* VBOX_STRICT */
1249
1250
1251/**
1252 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1253 * area.
1254 *
1255 * @returns VBox status code.
1256 * @param pVCpu The cross context virtual CPU structure.
1257 * @param cMsrs The number of MSRs.
1258 */
1259static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1260{
1261 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1262 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1263 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1264 {
1265 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1266 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1267 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1268 }
1269
1270 /* Update number of guest MSRs to load/store across the world-switch. */
1271 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1272 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1273
1274 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1275 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1276 AssertRCReturn(rc, rc);
1277
1278 /* Update the VCPU's copy of the MSR count. */
1279 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1280
1281 return VINF_SUCCESS;
1282}
1283
1284
1285/**
1286 * Adds a new (or updates the value of an existing) guest/host MSR
1287 * pair to be swapped during the world-switch as part of the
1288 * auto-load/store MSR area in the VMCS.
1289 *
1290 * @returns VBox status code.
1291 * @param pVCpu The cross context virtual CPU structure.
1292 * @param uMsr The MSR.
1293 * @param uGuestMsrValue Value of the guest MSR.
1294 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1295 * necessary.
1296 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1297 * its value was updated. Optional, can be NULL.
1298 */
1299static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1300 bool *pfAddedAndUpdated)
1301{
1302 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1303 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1304 uint32_t i;
1305 for (i = 0; i < cMsrs; i++)
1306 {
1307 if (pGuestMsr->u32Msr == uMsr)
1308 break;
1309 pGuestMsr++;
1310 }
1311
1312 bool fAdded = false;
1313 if (i == cMsrs)
1314 {
1315 ++cMsrs;
1316 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1317 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1318
1319 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1320 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1321 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1322
1323 fAdded = true;
1324 }
1325
1326 /* Update the MSR values in the auto-load/store MSR area. */
1327 pGuestMsr->u32Msr = uMsr;
1328 pGuestMsr->u64Value = uGuestMsrValue;
1329
1330 /* Create/update the MSR slot in the host MSR area. */
1331 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1332 pHostMsr += i;
1333 pHostMsr->u32Msr = uMsr;
1334
1335 /*
1336 * Update the host MSR only when requested by the caller AND when we're
1337 * adding it to the auto-load/store area. Otherwise, it would have been
1338 * updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1339 */
1340 bool fUpdatedMsrValue = false;
1341 if ( fAdded
1342 && fUpdateHostMsr)
1343 {
1344 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1345 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1346 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1347 fUpdatedMsrValue = true;
1348 }
1349
1350 if (pfAddedAndUpdated)
1351 *pfAddedAndUpdated = fUpdatedMsrValue;
1352 return VINF_SUCCESS;
1353}
1354
1355
1356/**
1357 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1358 * auto-load/store MSR area in the VMCS.
1359 *
1360 * @returns VBox status code.
1361 * @param pVCpu The cross context virtual CPU structure.
1362 * @param uMsr The MSR.
1363 */
1364static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1365{
1366 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1367 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1368 for (uint32_t i = 0; i < cMsrs; i++)
1369 {
1370 /* Find the MSR. */
1371 if (pGuestMsr->u32Msr == uMsr)
1372 {
1373 /* If it's the last MSR, simply reduce the count. */
1374 if (i == cMsrs - 1)
1375 {
1376 --cMsrs;
1377 break;
1378 }
1379
1380 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1381 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1382 pLastGuestMsr += cMsrs - 1;
1383 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1384 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1385
1386 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1387 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1388 pLastHostMsr += cMsrs - 1;
1389 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1390 pHostMsr->u64Value = pLastHostMsr->u64Value;
1391 --cMsrs;
1392 break;
1393 }
1394 pGuestMsr++;
1395 }
1396
1397 /* Update the VMCS if the count changed (meaning the MSR was found). */
1398 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1399 {
1400 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1401 AssertRCReturn(rc, rc);
1402
1403 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1404 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1405 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1406
1407 Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1408 return VINF_SUCCESS;
1409 }
1410
1411 return VERR_NOT_FOUND;
1412}
1413
1414
1415/**
1416 * Checks if the specified guest MSR is part of the auto-load/store area in
1417 * the VMCS.
1418 *
1419 * @returns true if found, false otherwise.
1420 * @param pVCpu The cross context virtual CPU structure.
1421 * @param uMsr The MSR to find.
1422 */
1423static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1424{
1425 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1426 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1427
1428 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1429 {
1430 if (pGuestMsr->u32Msr == uMsr)
1431 return true;
1432 }
1433 return false;
1434}
1435
1436
1437/**
1438 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1439 *
1440 * @param pVCpu The cross context virtual CPU structure.
1441 *
1442 * @remarks No-long-jump zone!!!
1443 */
1444static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1445{
1446 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1447 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1448 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1449 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1450
1451 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1452 {
1453 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1454
1455 /*
1456 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1457 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1458 */
1459 if (pHostMsr->u32Msr == MSR_K6_EFER)
1460 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1461 else
1462 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1463 }
1464
1465 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1466}
1467
1468
1469/**
1470 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1471 * perform lazy restoration of the host MSRs while leaving VT-x.
1472 *
1473 * @param pVCpu The cross context virtual CPU structure.
1474 *
1475 * @remarks No-long-jump zone!!!
1476 */
1477static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1478{
1479 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1480
1481 /*
1482 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1483 */
1484 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1485 {
1486 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1487#if HC_ARCH_BITS == 64
1488 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1489 {
1490 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1491 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1492 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1493 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1494 }
1495#endif
1496 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1497 }
1498}
1499
1500
1501/**
1502 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1503 * lazily while leaving VT-x.
1504 *
1505 * @returns true if it does, false otherwise.
1506 * @param pVCpu The cross context virtual CPU structure.
1507 * @param uMsr The MSR to check.
1508 */
1509static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1510{
1511 NOREF(pVCpu);
1512#if HC_ARCH_BITS == 64
1513 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1514 {
1515 switch (uMsr)
1516 {
1517 case MSR_K8_LSTAR:
1518 case MSR_K6_STAR:
1519 case MSR_K8_SF_MASK:
1520 case MSR_K8_KERNEL_GS_BASE:
1521 return true;
1522 }
1523 }
1524#else
1525 RT_NOREF(pVCpu, uMsr);
1526#endif
1527 return false;
1528}
1529
1530
1531/**
1532 * Loads a set of guests MSRs to allow read/passthru to the guest.
1533 *
1534 * The name of this function is slightly confusing. This function does NOT
1535 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1536 * common prefix for functions dealing with "lazy restoration" of the shared
1537 * MSRs.
1538 *
1539 * @param pVCpu The cross context virtual CPU structure.
1540 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1541 * out-of-sync. Make sure to update the required fields
1542 * before using them.
1543 *
1544 * @remarks No-long-jump zone!!!
1545 */
1546static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1547{
1548 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1549 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1550
1551 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1552#if HC_ARCH_BITS == 64
1553 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1554 {
1555 /*
1556 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1557 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1558 * we can skip a few MSR writes.
1559 *
1560 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1561 * guest MSR values in the guest-CPU context might be different to what's currently
1562 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1563 * CPU, see @bugref{8728}.
1564 */
1565 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1566 && pMixedCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1567 && pMixedCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1568 && pMixedCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1569 && pMixedCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1570 {
1571#ifdef VBOX_STRICT
1572 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pMixedCtx->msrKERNELGSBASE);
1573 Assert(ASMRdMsr(MSR_K8_LSTAR) == pMixedCtx->msrLSTAR);
1574 Assert(ASMRdMsr(MSR_K6_STAR) == pMixedCtx->msrSTAR);
1575 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pMixedCtx->msrSFMASK);
1576#endif
1577 }
1578 else
1579 {
1580 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1581 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1582 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1583 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1584 }
1585 }
1586#else
1587 RT_NOREF(pMixedCtx);
1588#endif
1589 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1590}
1591
1592
1593/**
1594 * Performs lazy restoration of the set of host MSRs if they were previously
1595 * loaded with guest MSR values.
1596 *
1597 * @param pVCpu The cross context virtual CPU structure.
1598 *
1599 * @remarks No-long-jump zone!!!
1600 * @remarks The guest MSRs should have been saved back into the guest-CPU
1601 * context by hmR0VmxImportGuestState()!!!
1602 */
1603static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1604{
1605 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1606 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1607
1608 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1609 {
1610 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1611#if HC_ARCH_BITS == 64
1612 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1613 {
1614 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1615 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1616 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1617 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1618 }
1619#endif
1620 }
1621 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1622}
1623
1624
1625/**
1626 * Verifies that our cached values of the VMCS controls are all
1627 * consistent with what's actually present in the VMCS.
1628 *
1629 * @returns VBox status code.
1630 * @param pVCpu The cross context virtual CPU structure.
1631 */
1632static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1633{
1634 uint32_t u32Val;
1635 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1636 AssertRCReturn(rc, rc);
1637 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1638 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1639
1640 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1641 AssertRCReturn(rc, rc);
1642 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1643 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1644
1645 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1646 AssertRCReturn(rc, rc);
1647 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1648 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1649
1650 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1651 AssertRCReturn(rc, rc);
1652 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1653 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1654
1655 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1656 {
1657 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1658 AssertRCReturn(rc, rc);
1659 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1660 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1661 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1662 }
1663
1664 return VINF_SUCCESS;
1665}
1666
1667
1668#ifdef VBOX_STRICT
1669/**
1670 * Verifies that our cached host EFER value has not changed
1671 * since we cached it.
1672 *
1673 * @param pVCpu The cross context virtual CPU structure.
1674 */
1675static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1676{
1677 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1678
1679 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1680 {
1681 uint64_t u64Val;
1682 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1683 AssertRC(rc);
1684
1685 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1686 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1687 }
1688}
1689
1690
1691/**
1692 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1693 * VMCS are correct.
1694 *
1695 * @param pVCpu The cross context virtual CPU structure.
1696 */
1697static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1698{
1699 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1700
1701 /* Verify MSR counts in the VMCS are what we think it should be. */
1702 uint32_t cMsrs;
1703 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1704 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1705
1706 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1707 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1708
1709 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1710 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1711
1712 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1713 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1714 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1715 {
1716 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1717 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1718 pGuestMsr->u32Msr, cMsrs));
1719
1720 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1721 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1722 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1723
1724 /* Verify that the permissions are as expected in the MSR bitmap. */
1725 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1726 {
1727 VMXMSREXITREAD enmRead;
1728 VMXMSREXITWRITE enmWrite;
1729 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1730 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1731 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1732 {
1733 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1734 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1735 }
1736 else
1737 {
1738 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1739 pGuestMsr->u32Msr, cMsrs));
1740 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1741 pGuestMsr->u32Msr, cMsrs));
1742 }
1743 }
1744 }
1745}
1746#endif /* VBOX_STRICT */
1747
1748
1749/**
1750 * Flushes the TLB using EPT.
1751 *
1752 * @returns VBox status code.
1753 * @param pVCpu The cross context virtual CPU structure of the calling
1754 * EMT. Can be NULL depending on @a enmFlush.
1755 * @param enmFlush Type of flush.
1756 *
1757 * @remarks Caller is responsible for making sure this function is called only
1758 * when NestedPaging is supported and providing @a enmFlush that is
1759 * supported by the CPU.
1760 * @remarks Can be called with interrupts disabled.
1761 */
1762static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1763{
1764 uint64_t au64Descriptor[2];
1765 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1766 au64Descriptor[0] = 0;
1767 else
1768 {
1769 Assert(pVCpu);
1770 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1771 }
1772 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1773
1774 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1775 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1776 rc));
1777 if ( RT_SUCCESS(rc)
1778 && pVCpu)
1779 {
1780 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1781 }
1782}
1783
1784
1785/**
1786 * Flushes the TLB using VPID.
1787 *
1788 * @returns VBox status code.
1789 * @param pVM The cross context VM structure.
1790 * @param pVCpu The cross context virtual CPU structure of the calling
1791 * EMT. Can be NULL depending on @a enmFlush.
1792 * @param enmFlush Type of flush.
1793 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1794 * on @a enmFlush).
1795 *
1796 * @remarks Can be called with interrupts disabled.
1797 */
1798static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1799{
1800 NOREF(pVM);
1801 AssertPtr(pVM);
1802 Assert(pVM->hm.s.vmx.fVpid);
1803
1804 uint64_t au64Descriptor[2];
1805 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1806 {
1807 au64Descriptor[0] = 0;
1808 au64Descriptor[1] = 0;
1809 }
1810 else
1811 {
1812 AssertPtr(pVCpu);
1813 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1814 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1815 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1816 au64Descriptor[1] = GCPtr;
1817 }
1818
1819 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1820 AssertMsg(rc == VINF_SUCCESS,
1821 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1822 if ( RT_SUCCESS(rc)
1823 && pVCpu)
1824 {
1825 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1826 }
1827}
1828
1829
1830/**
1831 * Invalidates a guest page by guest virtual address. Only relevant for
1832 * EPT/VPID, otherwise there is nothing really to invalidate.
1833 *
1834 * @returns VBox status code.
1835 * @param pVM The cross context VM structure.
1836 * @param pVCpu The cross context virtual CPU structure.
1837 * @param GCVirt Guest virtual address of the page to invalidate.
1838 */
1839VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1840{
1841 AssertPtr(pVM);
1842 AssertPtr(pVCpu);
1843 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1844
1845 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1846 if (!fFlushPending)
1847 {
1848 /*
1849 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1850 * the EPT case. See @bugref{6043} and @bugref{6177}.
1851 *
1852 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1853 * as this function maybe called in a loop with individual addresses.
1854 */
1855 if (pVM->hm.s.vmx.fVpid)
1856 {
1857 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1858
1859#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1860 /*
1861 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1862 * where executing INVVPID outside 64-bit mode does not flush translations of
1863 * 64-bit linear addresses, see @bugref{6208#c72}.
1864 */
1865 if (RT_HI_U32(GCVirt))
1866 fVpidFlush = false;
1867#endif
1868
1869 if (fVpidFlush)
1870 {
1871 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1872 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1873 }
1874 else
1875 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1876 }
1877 else if (pVM->hm.s.fNestedPaging)
1878 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1879 }
1880
1881 return VINF_SUCCESS;
1882}
1883
1884
1885/**
1886 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1887 * case where neither EPT nor VPID is supported by the CPU.
1888 *
1889 * @param pVM The cross context VM structure.
1890 * @param pVCpu The cross context virtual CPU structure.
1891 * @param pCpu Pointer to the global HM struct.
1892 *
1893 * @remarks Called with interrupts disabled.
1894 */
1895static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1896{
1897 AssertPtr(pVCpu);
1898 AssertPtr(pCpu);
1899 NOREF(pVM);
1900
1901 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1902
1903 Assert(pCpu->idCpu != NIL_RTCPUID);
1904 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1905 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1906 pVCpu->hm.s.fForceTLBFlush = false;
1907 return;
1908}
1909
1910
1911/**
1912 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1913 *
1914 * @param pVM The cross context VM structure.
1915 * @param pVCpu The cross context virtual CPU structure.
1916 * @param pCpu Pointer to the global HM CPU struct.
1917 * @remarks All references to "ASID" in this function pertains to "VPID" in
1918 * Intel's nomenclature. The reason is, to avoid confusion in compare
1919 * statements since the host-CPU copies are named "ASID".
1920 *
1921 * @remarks Called with interrupts disabled.
1922 */
1923static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1924{
1925#ifdef VBOX_WITH_STATISTICS
1926 bool fTlbFlushed = false;
1927# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1928# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1929 if (!fTlbFlushed) \
1930 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1931 } while (0)
1932#else
1933# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1934# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1935#endif
1936
1937 AssertPtr(pVM);
1938 AssertPtr(pCpu);
1939 AssertPtr(pVCpu);
1940 Assert(pCpu->idCpu != NIL_RTCPUID);
1941
1942 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1943 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1944 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1945
1946 /*
1947 * Force a TLB flush for the first world-switch if the current CPU differs from the one we
1948 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1949 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1950 * cannot reuse the current ASID anymore.
1951 */
1952 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1953 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1954 {
1955 ++pCpu->uCurrentAsid;
1956 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1957 {
1958 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1959 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1960 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1961 }
1962
1963 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1964 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1965 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1966
1967 /*
1968 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1969 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1970 */
1971 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1972 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1973 HMVMX_SET_TAGGED_TLB_FLUSHED();
1974 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1975 }
1976
1977 /* Check for explicit TLB flushes. */
1978 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1979 {
1980 /*
1981 * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
1982 * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
1983 * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
1984 * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
1985 * mappings, see @bugref{6568}.
1986 *
1987 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
1988 */
1989 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1990 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1991 HMVMX_SET_TAGGED_TLB_FLUSHED();
1992 }
1993
1994 pVCpu->hm.s.fForceTLBFlush = false;
1995 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1996
1997 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1998 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1999 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2000 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2001 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2002 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2003 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2004 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2005 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2006
2007 /* Update VMCS with the VPID. */
2008 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2009 AssertRC(rc);
2010
2011#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2012}
2013
2014
2015/**
2016 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2017 *
2018 * @returns VBox status code.
2019 * @param pVM The cross context VM structure.
2020 * @param pVCpu The cross context virtual CPU structure.
2021 * @param pCpu Pointer to the global HM CPU struct.
2022 *
2023 * @remarks Called with interrupts disabled.
2024 */
2025static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2026{
2027 AssertPtr(pVM);
2028 AssertPtr(pVCpu);
2029 AssertPtr(pCpu);
2030 Assert(pCpu->idCpu != NIL_RTCPUID);
2031 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2032 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2033
2034 /*
2035 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2036 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2037 */
2038 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2039 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2040 {
2041 pVCpu->hm.s.fForceTLBFlush = true;
2042 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2043 }
2044
2045 /* Check for explicit TLB flushes. */
2046 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2047 {
2048 pVCpu->hm.s.fForceTLBFlush = true;
2049 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2050 }
2051
2052 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2053 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2054
2055 if (pVCpu->hm.s.fForceTLBFlush)
2056 {
2057 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2058 pVCpu->hm.s.fForceTLBFlush = false;
2059 }
2060}
2061
2062
2063/**
2064 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2065 *
2066 * @returns VBox status code.
2067 * @param pVM The cross context VM structure.
2068 * @param pVCpu The cross context virtual CPU structure.
2069 * @param pCpu Pointer to the global HM CPU struct.
2070 *
2071 * @remarks Called with interrupts disabled.
2072 */
2073static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2074{
2075 AssertPtr(pVM);
2076 AssertPtr(pVCpu);
2077 AssertPtr(pCpu);
2078 Assert(pCpu->idCpu != NIL_RTCPUID);
2079 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2080 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2081
2082 /*
2083 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2084 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2085 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2086 */
2087 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2088 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2089 {
2090 pVCpu->hm.s.fForceTLBFlush = true;
2091 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2092 }
2093
2094 /* Check for explicit TLB flushes. */
2095 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2096 {
2097 /*
2098 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2099 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2100 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2101 */
2102 pVCpu->hm.s.fForceTLBFlush = true;
2103 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2104 }
2105
2106 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2107 if (pVCpu->hm.s.fForceTLBFlush)
2108 {
2109 ++pCpu->uCurrentAsid;
2110 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2111 {
2112 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2113 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2114 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2115 }
2116
2117 pVCpu->hm.s.fForceTLBFlush = false;
2118 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2119 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2120 if (pCpu->fFlushAsidBeforeUse)
2121 {
2122 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2123 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2124 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2125 {
2126 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2127 pCpu->fFlushAsidBeforeUse = false;
2128 }
2129 else
2130 {
2131 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2132 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2133 }
2134 }
2135 }
2136
2137 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2138 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2139 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2140 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2141 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2142 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2143 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2144
2145 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2146 AssertRC(rc);
2147}
2148
2149
2150/**
2151 * Flushes the guest TLB entry based on CPU capabilities.
2152 *
2153 * @param pVCpu The cross context virtual CPU structure.
2154 * @param pCpu Pointer to the global HM CPU struct.
2155 */
2156DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2157{
2158#ifdef HMVMX_ALWAYS_FLUSH_TLB
2159 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2160#endif
2161 PVM pVM = pVCpu->CTX_SUFF(pVM);
2162 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2163 {
2164 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2165 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2166 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2167 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2168 default:
2169 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2170 break;
2171 }
2172 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2173}
2174
2175
2176/**
2177 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2178 * TLB entries from the host TLB before VM-entry.
2179 *
2180 * @returns VBox status code.
2181 * @param pVM The cross context VM structure.
2182 */
2183static int hmR0VmxSetupTaggedTlb(PVM pVM)
2184{
2185 /*
2186 * Determine optimal flush type for Nested Paging.
2187 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2188 * guest execution (see hmR3InitFinalizeR0()).
2189 */
2190 if (pVM->hm.s.fNestedPaging)
2191 {
2192 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2193 {
2194 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2195 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2196 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2197 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2198 else
2199 {
2200 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2201 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2202 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2203 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2204 }
2205
2206 /* Make sure the write-back cacheable memory type for EPT is supported. */
2207 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2208 {
2209 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2210 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2211 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2212 }
2213
2214 /* EPT requires a page-walk length of 4. */
2215 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2216 {
2217 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2218 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2219 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2220 }
2221 }
2222 else
2223 {
2224 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2225 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2226 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2227 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2228 }
2229 }
2230
2231 /*
2232 * Determine optimal flush type for VPID.
2233 */
2234 if (pVM->hm.s.vmx.fVpid)
2235 {
2236 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2237 {
2238 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2239 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2240 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2241 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2242 else
2243 {
2244 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2245 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2246 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2247 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2248 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2249 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2250 pVM->hm.s.vmx.fVpid = false;
2251 }
2252 }
2253 else
2254 {
2255 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2256 Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2257 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2258 pVM->hm.s.vmx.fVpid = false;
2259 }
2260 }
2261
2262 /*
2263 * Setup the handler for flushing tagged-TLBs.
2264 */
2265 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2266 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2267 else if (pVM->hm.s.fNestedPaging)
2268 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2269 else if (pVM->hm.s.vmx.fVpid)
2270 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2271 else
2272 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2273 return VINF_SUCCESS;
2274}
2275
2276
2277/**
2278 * Sets up pin-based VM-execution controls in the VMCS.
2279 *
2280 * @returns VBox status code.
2281 * @param pVM The cross context VM structure.
2282 * @param pVCpu The cross context virtual CPU structure.
2283 */
2284static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2285{
2286 AssertPtr(pVM);
2287 AssertPtr(pVCpu);
2288
2289 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2290 uint32_t fZap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2291
2292 fVal |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2293 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2294
2295 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2296 fVal |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2297
2298 /* Enable the VMX preemption timer. */
2299 if (pVM->hm.s.vmx.fUsePreemptTimer)
2300 {
2301 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2302 fVal |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2303 }
2304
2305#if 0
2306 /* Enable posted-interrupt processing. */
2307 if (pVM->hm.s.fPostedIntrs)
2308 {
2309 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2310 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2311 fVal |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2312 }
2313#endif
2314
2315 if ((fVal & fZap) != fVal)
2316 {
2317 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2318 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, fVal, fZap));
2319 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2320 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2321 }
2322
2323 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2324 AssertRCReturn(rc, rc);
2325
2326 pVCpu->hm.s.vmx.u32PinCtls = fVal;
2327 return rc;
2328}
2329
2330
2331/**
2332 * Sets up processor-based VM-execution controls in the VMCS.
2333 *
2334 * @returns VBox status code.
2335 * @param pVM The cross context VM structure.
2336 * @param pVCpu The cross context virtual CPU structure.
2337 */
2338static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2339{
2340 AssertPtr(pVM);
2341 AssertPtr(pVCpu);
2342
2343 int rc = VERR_INTERNAL_ERROR_5;
2344 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2345 uint32_t fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2346
2347 fVal |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2348 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2349 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2350 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2351 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2352 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2353 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2354
2355 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2356 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2357 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2358 {
2359 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2360 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2361 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2362 }
2363
2364 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2365 if (!pVM->hm.s.fNestedPaging)
2366 {
2367 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2368 fVal |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2369 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2370 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2371 }
2372
2373 /* Use TPR shadowing if supported by the CPU. */
2374 if ( PDMHasApic(pVM)
2375 && pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2376 {
2377 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2378 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2379 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2380 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2381 AssertRCReturn(rc, rc);
2382
2383 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2384 /* CR8 writes cause a VM-exit based on TPR threshold. */
2385 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2386 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2387 }
2388 else
2389 {
2390 /*
2391 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2392 * Set this control only for 64-bit guests.
2393 */
2394 if (pVM->hm.s.fAllow64BitGuests)
2395 {
2396 fVal |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2397 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2398 }
2399 }
2400
2401 /* Use MSR-bitmaps if supported by the CPU. */
2402 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2403 {
2404 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2405
2406 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2407 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2408 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2409 AssertRCReturn(rc, rc);
2410
2411 /*
2412 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2413 * automatically using dedicated fields in the VMCS.
2414 */
2415 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2416 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2417 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2418 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2419 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2420
2421#if HC_ARCH_BITS == 64
2422 /*
2423 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2424 */
2425 if (pVM->hm.s.fAllow64BitGuests)
2426 {
2427 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2428 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2429 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2430 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2431 }
2432#endif
2433 /*
2434 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2435 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2436 */
2437 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2438 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2439
2440 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2441 }
2442
2443 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2444 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2445 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2446
2447 if ((fVal & fZap) != fVal)
2448 {
2449 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2450 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, fVal, fZap));
2451 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2452 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2453 }
2454
2455 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2456 AssertRCReturn(rc, rc);
2457
2458 pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2459
2460 /*
2461 * Secondary processor-based VM-execution controls.
2462 */
2463 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2464 {
2465 fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2466 fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2467
2468 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2469 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2470
2471 if (pVM->hm.s.fNestedPaging)
2472 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2473
2474 /*
2475 * Enable the INVPCID instruction if supported by the hardware and we expose
2476 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2477 */
2478 if ( (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2479 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2480 {
2481 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2482 }
2483
2484 if (pVM->hm.s.vmx.fVpid)
2485 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2486
2487 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2488 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2489
2490#if 0
2491 if (pVM->hm.s.fVirtApicRegs)
2492 {
2493 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2494 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2495
2496 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2497 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2498 }
2499#endif
2500
2501 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2502 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2503 * done dynamically. */
2504 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2505 {
2506 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2507 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2508 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2509 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2510 AssertRCReturn(rc, rc);
2511 }
2512
2513 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2514 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2515
2516 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2517 && pVM->hm.s.vmx.cPleGapTicks
2518 && pVM->hm.s.vmx.cPleWindowTicks)
2519 {
2520 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2521
2522 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2523 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2524 AssertRCReturn(rc, rc);
2525 }
2526
2527 if ((fVal & fZap) != fVal)
2528 {
2529 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2530 "cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, fVal, fZap));
2531 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2532 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2533 }
2534
2535 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2536 AssertRCReturn(rc, rc);
2537
2538 pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2539 }
2540 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2541 {
2542 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2543 "available\n"));
2544 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2545 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2546 }
2547
2548 return VINF_SUCCESS;
2549}
2550
2551
2552/**
2553 * Sets up miscellaneous (everything other than Pin & Processor-based
2554 * VM-execution) control fields in the VMCS.
2555 *
2556 * @returns VBox status code.
2557 * @param pVM The cross context VM structure.
2558 * @param pVCpu The cross context virtual CPU structure.
2559 */
2560static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2561{
2562 NOREF(pVM);
2563 AssertPtr(pVM);
2564 AssertPtr(pVCpu);
2565
2566 int rc = VERR_GENERAL_FAILURE;
2567
2568 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2569#if 0
2570 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2571 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2572 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2573
2574 /*
2575 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2576 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2577 * We thus use the exception bitmap to control it rather than use both.
2578 */
2579 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2580 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2581
2582 /* All IO & IOIO instructions cause VM-exits. */
2583 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2584 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2585
2586 /* Initialize the MSR-bitmap area. */
2587 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2588 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2589 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2590 AssertRCReturn(rc, rc);
2591#endif
2592
2593 /* Setup MSR auto-load/store area. */
2594 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2595 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2596 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2597 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2598 AssertRCReturn(rc, rc);
2599
2600 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2601 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2602 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2603 AssertRCReturn(rc, rc);
2604
2605 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2606 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2607 AssertRCReturn(rc, rc);
2608
2609 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2610#if 0
2611 /* Setup debug controls */
2612 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2613 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2614 AssertRCReturn(rc, rc);
2615#endif
2616
2617 return rc;
2618}
2619
2620
2621/**
2622 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2623 *
2624 * We shall setup those exception intercepts that don't change during the
2625 * lifetime of the VM here. The rest are done dynamically while loading the
2626 * guest state.
2627 *
2628 * @returns VBox status code.
2629 * @param pVM The cross context VM structure.
2630 * @param pVCpu The cross context virtual CPU structure.
2631 */
2632static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2633{
2634 AssertPtr(pVM);
2635 AssertPtr(pVCpu);
2636
2637 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2638
2639 uint32_t u32XcptBitmap = 0;
2640
2641 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2642 u32XcptBitmap |= RT_BIT_32(X86_XCPT_AC);
2643
2644 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2645 and writes, and because recursive #DBs can cause the CPU hang, we must always
2646 intercept #DB. */
2647 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2648
2649 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2650 if (!pVM->hm.s.fNestedPaging)
2651 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2652
2653 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2654 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2655 AssertRCReturn(rc, rc);
2656 return rc;
2657}
2658
2659
2660/**
2661 * Does per-VM VT-x initialization.
2662 *
2663 * @returns VBox status code.
2664 * @param pVM The cross context VM structure.
2665 */
2666VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2667{
2668 LogFlowFunc(("pVM=%p\n", pVM));
2669
2670 int rc = hmR0VmxStructsAlloc(pVM);
2671 if (RT_FAILURE(rc))
2672 {
2673 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2674 return rc;
2675 }
2676
2677 return VINF_SUCCESS;
2678}
2679
2680
2681/**
2682 * Does per-VM VT-x termination.
2683 *
2684 * @returns VBox status code.
2685 * @param pVM The cross context VM structure.
2686 */
2687VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2688{
2689 LogFlowFunc(("pVM=%p\n", pVM));
2690
2691#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2692 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2693 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2694#endif
2695 hmR0VmxStructsFree(pVM);
2696 return VINF_SUCCESS;
2697}
2698
2699
2700/**
2701 * Sets up the VM for execution under VT-x.
2702 * This function is only called once per-VM during initialization.
2703 *
2704 * @returns VBox status code.
2705 * @param pVM The cross context VM structure.
2706 */
2707VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2708{
2709 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2710 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2711
2712 LogFlowFunc(("pVM=%p\n", pVM));
2713
2714 /*
2715 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be
2716 * allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2717 * pRealModeTSS, see hmR3InitFinalizeR0Intel().
2718 */
2719 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2720 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2721 || !pVM->hm.s.vmx.pRealModeTSS))
2722 {
2723 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2724 return VERR_INTERNAL_ERROR;
2725 }
2726
2727 /* Initialize these always, see hmR3InitFinalizeR0().*/
2728 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2729 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2730
2731 /* Setup the tagged-TLB flush handlers. */
2732 int rc = hmR0VmxSetupTaggedTlb(pVM);
2733 if (RT_FAILURE(rc))
2734 {
2735 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2736 return rc;
2737 }
2738
2739 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2740 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2741#if HC_ARCH_BITS == 64
2742 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2743 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2744 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2745 {
2746 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2747 }
2748#endif
2749
2750 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2751 RTCCUINTREG uHostCR4 = ASMGetCR4();
2752 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2753 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2754
2755 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2756 {
2757 PVMCPU pVCpu = &pVM->aCpus[i];
2758 AssertPtr(pVCpu);
2759 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2760
2761 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2762 Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2763
2764 /* Set revision dword at the beginning of the VMCS structure. */
2765 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2766
2767 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2768 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2769 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2770 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2771
2772 /* Load this VMCS as the current VMCS. */
2773 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2774 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2775 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2776
2777 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2778 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2779 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2780
2781 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2782 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2783 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2784
2785 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2786 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2787 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2788
2789 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2790 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2791 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2792
2793#if HC_ARCH_BITS == 32
2794 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2795 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2796 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2797#endif
2798
2799 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2800 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2801 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2802 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2803
2804 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2805
2806 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2807 }
2808
2809 return VINF_SUCCESS;
2810}
2811
2812
2813/**
2814 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2815 * the VMCS.
2816 *
2817 * @returns VBox status code.
2818 */
2819static int hmR0VmxExportHostControlRegs(void)
2820{
2821 RTCCUINTREG uReg = ASMGetCR0();
2822 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2823 AssertRCReturn(rc, rc);
2824
2825 uReg = ASMGetCR3();
2826 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2827 AssertRCReturn(rc, rc);
2828
2829 uReg = ASMGetCR4();
2830 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2831 AssertRCReturn(rc, rc);
2832 return rc;
2833}
2834
2835
2836/**
2837 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2838 * the host-state area in the VMCS.
2839 *
2840 * @returns VBox status code.
2841 * @param pVCpu The cross context virtual CPU structure.
2842 */
2843static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2844{
2845#if HC_ARCH_BITS == 64
2846/**
2847 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2848 * requirements. See hmR0VmxExportHostSegmentRegs().
2849 */
2850# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2851 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2852 { \
2853 bool fValidSelector = true; \
2854 if ((selValue) & X86_SEL_LDT) \
2855 { \
2856 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2857 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2858 } \
2859 if (fValidSelector) \
2860 { \
2861 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2862 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2863 } \
2864 (selValue) = 0; \
2865 }
2866
2867 /*
2868 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2869 * should -not- save the messed up state without restoring the original host-state,
2870 * see @bugref{7240}.
2871 *
2872 * This apparently can happen (most likely the FPU changes), deal with it rather than
2873 * asserting. Was observed booting Solaris 10u10 32-bit guest.
2874 */
2875 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2876 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2877 {
2878 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2879 pVCpu->idCpu));
2880 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2881 }
2882 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2883#else
2884 RT_NOREF(pVCpu);
2885#endif
2886
2887 /*
2888 * Host DS, ES, FS and GS segment registers.
2889 */
2890#if HC_ARCH_BITS == 64
2891 RTSEL uSelDS = ASMGetDS();
2892 RTSEL uSelES = ASMGetES();
2893 RTSEL uSelFS = ASMGetFS();
2894 RTSEL uSelGS = ASMGetGS();
2895#else
2896 RTSEL uSelDS = 0;
2897 RTSEL uSelES = 0;
2898 RTSEL uSelFS = 0;
2899 RTSEL uSelGS = 0;
2900#endif
2901
2902 /*
2903 * Host CS and SS segment registers.
2904 */
2905 RTSEL uSelCS = ASMGetCS();
2906 RTSEL uSelSS = ASMGetSS();
2907
2908 /*
2909 * Host TR segment register.
2910 */
2911 RTSEL uSelTR = ASMGetTR();
2912
2913#if HC_ARCH_BITS == 64
2914 /*
2915 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2916 * gain VM-entry and restore them before we get preempted.
2917 *
2918 * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2919 */
2920 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2921 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2922 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2923 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2924# undef VMXLOCAL_ADJUST_HOST_SEG
2925#endif
2926
2927 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2928 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2929 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2930 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2931 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2932 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2933 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2934 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2935 Assert(uSelCS);
2936 Assert(uSelTR);
2937
2938 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2939#if 0
2940 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2941 Assert(uSelSS != 0);
2942#endif
2943
2944 /* Write these host selector fields into the host-state area in the VMCS. */
2945 int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2946 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2947#if HC_ARCH_BITS == 64
2948 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2949 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2950 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2951 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2952#else
2953 NOREF(uSelDS);
2954 NOREF(uSelES);
2955 NOREF(uSelFS);
2956 NOREF(uSelGS);
2957#endif
2958 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2959 AssertRCReturn(rc, rc);
2960
2961 /*
2962 * Host GDTR and IDTR.
2963 */
2964 RTGDTR Gdtr;
2965 RTIDTR Idtr;
2966 RT_ZERO(Gdtr);
2967 RT_ZERO(Idtr);
2968 ASMGetGDTR(&Gdtr);
2969 ASMGetIDTR(&Idtr);
2970 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2971 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2972 AssertRCReturn(rc, rc);
2973
2974#if HC_ARCH_BITS == 64
2975 /*
2976 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
2977 * them to the maximum limit (0xffff) on every VM-exit.
2978 */
2979 if (Gdtr.cbGdt != 0xffff)
2980 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2981
2982 /*
2983 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
2984 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the
2985 * limit as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU
2986 * behavior. However, several hosts either insists on 0xfff being the limit (Windows
2987 * Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
2988 * but botches sidt alignment in at least one consumer). So, we're only allowing the
2989 * IDTR.LIMIT to be left at 0xffff on hosts where we are sure it won't cause trouble.
2990 */
2991# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
2992 if (Idtr.cbIdt < 0x0fff)
2993# else
2994 if (Idtr.cbIdt != 0xffff)
2995# endif
2996 {
2997 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
2998 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2999 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3000 }
3001#endif
3002
3003 /*
3004 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3005 * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3006 * RPL should be too in most cases.
3007 */
3008 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3009 ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3010
3011 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3012#if HC_ARCH_BITS == 64
3013 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3014
3015 /*
3016 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3017 * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3018 * restoration if the host has something else. Task switching is not supported in 64-bit
3019 * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3020 * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3021 *
3022 * [1] See Intel spec. 3.5 "System Descriptor Types".
3023 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3024 */
3025 PVM pVM = pVCpu->CTX_SUFF(pVM);
3026 Assert(pDesc->System.u4Type == 11);
3027 if ( pDesc->System.u16LimitLow != 0x67
3028 || pDesc->System.u4LimitHigh)
3029 {
3030 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3031 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3032 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3033 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3034 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3035 }
3036
3037 /*
3038 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3039 */
3040 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3041 {
3042 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3043 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3044 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3045 {
3046 /* The GDT is read-only but the writable GDT is available. */
3047 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3048 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3049 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3050 AssertRCReturn(rc, rc);
3051 }
3052 }
3053#else
3054 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3055#endif
3056 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3057 AssertRCReturn(rc, rc);
3058
3059 /*
3060 * Host FS base and GS base.
3061 */
3062#if HC_ARCH_BITS == 64
3063 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3064 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3065 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3066 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3067 AssertRCReturn(rc, rc);
3068
3069 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3070 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3071 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3072 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3073 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3074#endif
3075 return VINF_SUCCESS;
3076}
3077
3078
3079/**
3080 * Exports certain host MSRs in the VM-exit MSR-load area and some in the
3081 * host-state area of the VMCS.
3082 *
3083 * Theses MSRs will be automatically restored on the host after every successful
3084 * VM-exit.
3085 *
3086 * @returns VBox status code.
3087 * @param pVCpu The cross context virtual CPU structure.
3088 *
3089 * @remarks No-long-jump zone!!!
3090 */
3091static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3092{
3093 AssertPtr(pVCpu);
3094 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3095
3096 /*
3097 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3098 * rather than swapping them on every VM-entry.
3099 */
3100 hmR0VmxLazySaveHostMsrs(pVCpu);
3101
3102 /*
3103 * Host Sysenter MSRs.
3104 */
3105 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3106#if HC_ARCH_BITS == 32
3107 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3108 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3109#else
3110 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3111 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3112#endif
3113 AssertRCReturn(rc, rc);
3114
3115 /*
3116 * Host EFER MSR.
3117 *
3118 * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3119 * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3120 */
3121 PVM pVM = pVCpu->CTX_SUFF(pVM);
3122 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3123 {
3124 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3125 AssertRCReturn(rc, rc);
3126 }
3127
3128 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3129
3130 return VINF_SUCCESS;
3131}
3132
3133
3134/**
3135 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3136 *
3137 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3138 * these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3139 * hmR0VMxExportGuestEntryCtls().
3140 *
3141 * @returns true if we need to load guest EFER, false otherwise.
3142 * @param pVCpu The cross context virtual CPU structure.
3143 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3144 * out-of-sync. Make sure to update the required fields
3145 * before using them.
3146 *
3147 * @remarks Requires EFER, CR4.
3148 * @remarks No-long-jump zone!!!
3149 */
3150static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3151{
3152#ifdef HMVMX_ALWAYS_SWAP_EFER
3153 return true;
3154#endif
3155
3156#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3157 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3158 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3159 return false;
3160#endif
3161
3162 PVM pVM = pVCpu->CTX_SUFF(pVM);
3163 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3164 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3165
3166 /*
3167 * For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3168 * guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3169 */
3170 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3171 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3172 {
3173 return true;
3174 }
3175
3176 /*
3177 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3178 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3179 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3180 */
3181 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3182 && (pMixedCtx->cr0 & X86_CR0_PG)
3183 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3184 {
3185 /* Assert that host is PAE capable. */
3186 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3187 return true;
3188 }
3189
3190 return false;
3191}
3192
3193
3194/**
3195 * Exports the guest state with appropriate VM-entry controls in the VMCS.
3196 *
3197 * These controls can affect things done on VM-exit; e.g. "load debug controls",
3198 * see Intel spec. 24.8.1 "VM-entry controls".
3199 *
3200 * @returns VBox status code.
3201 * @param pVCpu The cross context virtual CPU structure.
3202 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3203 * out-of-sync. Make sure to update the required fields
3204 * before using them.
3205 *
3206 * @remarks Requires EFER.
3207 * @remarks No-long-jump zone!!!
3208 */
3209static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3210{
3211 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3212 {
3213 PVM pVM = pVCpu->CTX_SUFF(pVM);
3214 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3215 uint32_t fZap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3216
3217 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3218 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3219
3220 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3221 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3222 {
3223 fVal |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3224 Log4Func(("VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n"));
3225 }
3226 else
3227 Assert(!(fVal & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3228
3229 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3230 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3231 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3232 {
3233 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3234 Log4Func(("VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n"));
3235 }
3236
3237 /*
3238 * The following should -not- be set (since we're not in SMM mode):
3239 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3240 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3241 */
3242
3243 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3244 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3245
3246 if ((fVal & fZap) != fVal)
3247 {
3248 Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3249 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, fVal, fZap));
3250 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3251 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3252 }
3253
3254 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3255 AssertRCReturn(rc, rc);
3256
3257 pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3258 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3259 }
3260 return VINF_SUCCESS;
3261}
3262
3263
3264/**
3265 * Exports the guest state with appropriate VM-exit controls in the VMCS.
3266 *
3267 * @returns VBox status code.
3268 * @param pVCpu The cross context virtual CPU structure.
3269 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3270 * out-of-sync. Make sure to update the required fields
3271 * before using them.
3272 *
3273 * @remarks Requires EFER.
3274 */
3275static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3276{
3277 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3278 {
3279 PVM pVM = pVCpu->CTX_SUFF(pVM);
3280 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3281 uint32_t fZap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3282
3283 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3284 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3285
3286 /*
3287 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3288 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3289 * hmR0VmxExportHostMsrs().
3290 */
3291#if HC_ARCH_BITS == 64
3292 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3293 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3294#else
3295 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3296 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3297 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3298 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3299 {
3300 /* The switcher returns to long mode, EFER is managed by the switcher. */
3301 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3302 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3303 }
3304 else
3305 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3306#endif
3307
3308 /* If the newer VMCS fields for managing EFER exists, use it. */
3309 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3310 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3311 {
3312 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3313 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3314 Log4Func(("VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR and VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n"));
3315 }
3316
3317 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3318 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3319
3320 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3321 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3322 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3323
3324 if ( pVM->hm.s.vmx.fUsePreemptTimer
3325 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3326 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3327
3328 if ((fVal & fZap) != fVal)
3329 {
3330 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3331 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, fVal, fZap));
3332 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3333 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3334 }
3335
3336 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3337 AssertRCReturn(rc, rc);
3338
3339 pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3340 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3341 }
3342 return VINF_SUCCESS;
3343}
3344
3345
3346/**
3347 * Sets the TPR threshold in the VMCS.
3348 *
3349 * @returns VBox status code.
3350 * @param pVCpu The cross context virtual CPU structure.
3351 * @param u32TprThreshold The TPR threshold (task-priority class only).
3352 */
3353DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3354{
3355 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3356 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3357 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3358}
3359
3360
3361/**
3362 * Exports the guest APIC TPR state into the VMCS.
3363 *
3364 * @returns VBox status code.
3365 * @param pVCpu The cross context virtual CPU structure.
3366 *
3367 * @remarks No-long-jump zone!!!
3368 */
3369static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3370{
3371 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3372 {
3373 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3374 && APICIsEnabled(pVCpu))
3375 {
3376 /*
3377 * Setup TPR shadowing.
3378 */
3379 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3380 {
3381 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3382
3383 bool fPendingIntr = false;
3384 uint8_t u8Tpr = 0;
3385 uint8_t u8PendingIntr = 0;
3386 int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3387 AssertRCReturn(rc, rc);
3388
3389 /*
3390 * If there are interrupts pending but masked by the TPR, instruct VT-x to
3391 * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3392 * priority of the pending interrupt so we can deliver the interrupt. If there
3393 * are no interrupts pending, set threshold to 0 to not cause any
3394 * TPR-below-threshold VM-exits.
3395 */
3396 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3397 uint32_t u32TprThreshold = 0;
3398 if (fPendingIntr)
3399 {
3400 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3401 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3402 const uint8_t u8TprPriority = u8Tpr >> 4;
3403 if (u8PendingPriority <= u8TprPriority)
3404 u32TprThreshold = u8PendingPriority;
3405 }
3406
3407 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3408 AssertRCReturn(rc, rc);
3409 }
3410 }
3411 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3412 }
3413 return VINF_SUCCESS;
3414}
3415
3416
3417/**
3418 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3419 *
3420 * @returns Guest's interruptibility-state.
3421 * @param pVCpu The cross context virtual CPU structure.
3422 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3423 * out-of-sync. Make sure to update the required fields
3424 * before using them.
3425 *
3426 * @remarks No-long-jump zone!!!
3427 */
3428static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3429{
3430 /*
3431 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3432 */
3433 uint32_t fIntrState = 0;
3434 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3435 {
3436 /* If inhibition is active, RIP & RFLAGS should've been accessed
3437 (i.e. read previously from the VMCS or from ring-3). */
3438#ifdef VBOX_STRICT
3439 uint64_t const fExtrn = ASMAtomicUoReadU64(&pMixedCtx->fExtrn);
3440 AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3441#endif
3442 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3443 {
3444 if (pMixedCtx->eflags.Bits.u1IF)
3445 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3446 else
3447 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3448 }
3449 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3450 {
3451 /*
3452 * We can clear the inhibit force flag as even if we go back to the recompiler
3453 * without executing guest code in VT-x, the flag's condition to be cleared is
3454 * met and thus the cleared state is correct.
3455 */
3456 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3457 }
3458 }
3459
3460 /*
3461 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3462 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3463 * setting this would block host-NMIs and IRET will not clear the blocking.
3464 *
3465 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3466 */
3467 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3468 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3469 {
3470 fIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3471 }
3472
3473 return fIntrState;
3474}
3475
3476
3477/**
3478 * Exports the guest's interruptibility-state into the guest-state area in the
3479 * VMCS.
3480 *
3481 * @returns VBox status code.
3482 * @param pVCpu The cross context virtual CPU structure.
3483 * @param fIntrState The interruptibility-state to set.
3484 */
3485static int hmR0VmxExportGuestIntrState(PVMCPU pVCpu, uint32_t fIntrState)
3486{
3487 NOREF(pVCpu);
3488 AssertMsg(!(fIntrState & 0xfffffff0), ("%#x\n", fIntrState)); /* Bits 31:4 MBZ. */
3489 Assert((fIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3490 return VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, fIntrState);
3491}
3492
3493
3494/**
3495 * Exports the exception intercepts required for guest execution in the VMCS.
3496 *
3497 * @returns VBox status code.
3498 * @param pVCpu The cross context virtual CPU structure.
3499 *
3500 * @remarks No-long-jump zone!!!
3501 */
3502static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3503{
3504 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3505 {
3506 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3507 if (pVCpu->hm.s.fGIMTrapXcptUD)
3508 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3509#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3510 else
3511 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3512#endif
3513
3514 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3515 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3516
3517 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3518 AssertRCReturn(rc, rc);
3519
3520 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3521 Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", pVCpu->hm.s.vmx.u32XcptBitmap));
3522 }
3523 return VINF_SUCCESS;
3524}
3525
3526
3527/**
3528 * Exports the guest's RIP into the guest-state area in the VMCS.
3529 *
3530 * @returns VBox status code.
3531 * @param pVCpu The cross context virtual CPU structure.
3532 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3533 * out-of-sync. Make sure to update the required fields
3534 * before using them.
3535 *
3536 * @remarks No-long-jump zone!!!
3537 */
3538static int hmR0VmxExportGuestRip(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3539{
3540 int rc = VINF_SUCCESS;
3541 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3542 {
3543 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3544 AssertRCReturn(rc, rc);
3545
3546 /* Update the exit history entry with the correct CS.BASE + RIP or just RIP. */
3547 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
3548 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
3549 else
3550 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->rip, false);
3551
3552 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3553 Log4Func(("RIP=%#RX64\n", pMixedCtx->rip));
3554 }
3555 return rc;
3556}
3557
3558
3559/**
3560 * Exports the guest's RSP into the guest-state area in the VMCS.
3561 *
3562 * @returns VBox status code.
3563 * @param pVCpu The cross context virtual CPU structure.
3564 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3565 * out-of-sync. Make sure to update the required fields
3566 * before using them.
3567 *
3568 * @remarks No-long-jump zone!!!
3569 */
3570static int hmR0VmxExportGuestRsp(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3571{
3572 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3573 {
3574 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3575 AssertRCReturn(rc, rc);
3576
3577 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3578 }
3579 return VINF_SUCCESS;
3580}
3581
3582
3583/**
3584 * Exports the guest's RFLAGS into the guest-state area in the VMCS.
3585 *
3586 * @returns VBox status code.
3587 * @param pVCpu The cross context virtual CPU structure.
3588 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3589 * out-of-sync. Make sure to update the required fields
3590 * before using them.
3591 *
3592 * @remarks No-long-jump zone!!!
3593 */
3594static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3595{
3596 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3597 {
3598 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3599 Let us assert it as such and use 32-bit VMWRITE. */
3600 Assert(!(pMixedCtx->rflags.u64 >> 32));
3601 X86EFLAGS fEFlags = pMixedCtx->eflags;
3602 Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3603 Assert(!(fEFlags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3604
3605 /*
3606 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3607 * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3608 * can run the real-mode guest code under Virtual 8086 mode.
3609 */
3610 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3611 {
3612 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3613 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3614 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3615 fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3616 fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3617 }
3618
3619 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3620 AssertRCReturn(rc, rc);
3621
3622 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3623 Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3624 }
3625 return VINF_SUCCESS;
3626}
3627
3628
3629/**
3630 * Exports the guest CR0 control register into the guest-state area in the VMCS.
3631 *
3632 * The guest FPU state is always pre-loaded hence we don't need to bother about
3633 * sharing FPU related CR0 bits between the guest and host.
3634 *
3635 * @returns VBox status code.
3636 * @param pVCpu The cross context virtual CPU structure.
3637 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3638 * out-of-sync. Make sure to update the required fields
3639 * before using them.
3640 *
3641 * @remarks No-long-jump zone!!!
3642 */
3643static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3644{
3645 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3646 {
3647 PVM pVM = pVCpu->CTX_SUFF(pVM);
3648 Assert(!RT_HI_U32(pMixedCtx->cr0));
3649 uint32_t const uShadowCR0 = pMixedCtx->cr0;
3650 uint32_t uGuestCR0 = pMixedCtx->cr0;
3651
3652 /*
3653 * Setup VT-x's view of the guest CR0.
3654 * Minimize VM-exits due to CR3 changes when we have NestedPaging.
3655 */
3656 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3657 if (pVM->hm.s.fNestedPaging)
3658 {
3659 if (CPUMIsGuestPagingEnabled(pVCpu))
3660 {
3661 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3662 uProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3663 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3664 }
3665 else
3666 {
3667 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3668 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3669 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3670 }
3671
3672 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3673 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3674 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3675 }
3676 else
3677 {
3678 /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3679 uGuestCR0 |= X86_CR0_WP;
3680 }
3681
3682 /*
3683 * Guest FPU bits.
3684 *
3685 * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3686 * using CR0.TS.
3687 *
3688 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3689 * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3690 */
3691 uGuestCR0 |= X86_CR0_NE;
3692
3693 /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3694 bool const fInterceptMF = !(uShadowCR0 & X86_CR0_NE);
3695
3696 /*
3697 * Update exception intercepts.
3698 */
3699 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3700 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3701 {
3702 Assert(PDMVmmDevHeapIsEnabled(pVM));
3703 Assert(pVM->hm.s.vmx.pRealModeTSS);
3704 uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3705 }
3706 else
3707 {
3708 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3709 uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3710 if (fInterceptMF)
3711 uXcptBitmap |= RT_BIT(X86_XCPT_MF);
3712 }
3713
3714 /* Additional intercepts for debugging, define these yourself explicitly. */
3715#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3716 uXcptBitmap |= 0
3717 | RT_BIT(X86_XCPT_BP)
3718 | RT_BIT(X86_XCPT_DE)
3719 | RT_BIT(X86_XCPT_NM)
3720 | RT_BIT(X86_XCPT_TS)
3721 | RT_BIT(X86_XCPT_UD)
3722 | RT_BIT(X86_XCPT_NP)
3723 | RT_BIT(X86_XCPT_SS)
3724 | RT_BIT(X86_XCPT_GP)
3725 | RT_BIT(X86_XCPT_PF)
3726 | RT_BIT(X86_XCPT_MF)
3727 ;
3728#elif defined(HMVMX_ALWAYS_TRAP_PF)
3729 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
3730#endif
3731 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3732 {
3733 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3734 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3735 }
3736 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3737
3738 /*
3739 * Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3740 */
3741 uint32_t fSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3742 uint32_t fZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3743 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3744 fSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3745 else
3746 Assert((fSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3747
3748 uGuestCR0 |= fSetCR0;
3749 uGuestCR0 &= fZapCR0;
3750 uGuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3751
3752 /*
3753 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3754 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3755 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3756 */
3757 uint32_t uCR0Mask = X86_CR0_PE
3758 | X86_CR0_NE
3759 | (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3760 | X86_CR0_PG
3761 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3762 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3763 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3764
3765 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3766 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3767 * and @bugref{6944}. */
3768#if 0
3769 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3770 uCr0Mask &= ~X86_CR0_PE;
3771#endif
3772 /* Update the HMCPU's copy of the CR0 mask. */
3773 pVCpu->hm.s.vmx.u32CR0Mask = uCR0Mask;
3774
3775 /*
3776 * Finally, update VMCS fields with the CR0 values.
3777 */
3778 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, uGuestCR0);
3779 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, uShadowCR0);
3780 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, uCR0Mask);
3781 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3782 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3783 AssertRCReturn(rc, rc);
3784 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3785
3786 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3787
3788 Log4Func(("uCr0Mask=%#RX32 uShadowCR0=%#RX32 uGuestCR0=%#RX32 (fSetCR0=%#RX32 fZapCR0=%#RX32\n", uCR0Mask, uShadowCR0,
3789 uGuestCR0, fSetCR0, fZapCR0));
3790 }
3791
3792 return VINF_SUCCESS;
3793}
3794
3795
3796/**
3797 * Exports the guest control registers (CR3, CR4) into the guest-state area
3798 * in the VMCS.
3799 *
3800 * @returns VBox strict status code.
3801 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3802 * without unrestricted guest access and the VMMDev is not presently
3803 * mapped (e.g. EFI32).
3804 *
3805 * @param pVCpu The cross context virtual CPU structure.
3806 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3807 * out-of-sync. Make sure to update the required fields
3808 * before using them.
3809 *
3810 * @remarks No-long-jump zone!!!
3811 */
3812static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3813{
3814 int rc = VINF_SUCCESS;
3815 PVM pVM = pVCpu->CTX_SUFF(pVM);
3816
3817 /*
3818 * Guest CR2.
3819 * It's always loaded in the assembler code. Nothing to do here.
3820 */
3821
3822 /*
3823 * Guest CR3.
3824 */
3825 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3826 {
3827 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3828 if (pVM->hm.s.fNestedPaging)
3829 {
3830 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3831
3832 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3833 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3834 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3835 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3836
3837 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3838 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3839 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3840
3841 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3842 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3843 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3844 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3845 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3846 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3847 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3848
3849 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3850 AssertRCReturn(rc, rc);
3851
3852 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3853 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3854 {
3855 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3856 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3857 {
3858 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3859 AssertRCReturn(rc, rc);
3860 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3861 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3862 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3863 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3864 AssertRCReturn(rc, rc);
3865 }
3866
3867 /*
3868 * The guest's view of its CR3 is unblemished with Nested Paging when the
3869 * guest is using paging or we have unrestricted guest execution to handle
3870 * the guest when it's not using paging.
3871 */
3872 GCPhysGuestCR3 = pMixedCtx->cr3;
3873 }
3874 else
3875 {
3876 /*
3877 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
3878 * thinks it accesses physical memory directly, we use our identity-mapped
3879 * page table to map guest-linear to guest-physical addresses. EPT takes care
3880 * of translating it to host-physical addresses.
3881 */
3882 RTGCPHYS GCPhys;
3883 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3884
3885 /* We obtain it here every time as the guest could have relocated this PCI region. */
3886 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3887 if (RT_SUCCESS(rc))
3888 { /* likely */ }
3889 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3890 {
3891 Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3892 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3893 }
3894 else
3895 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3896
3897 GCPhysGuestCR3 = GCPhys;
3898 }
3899
3900 Log4Func(("uGuestCR3=%#RGp (GstN)\n", GCPhysGuestCR3));
3901 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3902 AssertRCReturn(rc, rc);
3903 }
3904 else
3905 {
3906 /* Non-nested paging case, just use the hypervisor's CR3. */
3907 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3908
3909 Log4Func(("uGuestCR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3910 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3911 AssertRCReturn(rc, rc);
3912 }
3913
3914 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3915 }
3916
3917 /*
3918 * Guest CR4.
3919 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3920 */
3921 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3922 {
3923 Assert(!RT_HI_U32(pMixedCtx->cr4));
3924 uint32_t uGuestCR4 = pMixedCtx->cr4;
3925
3926 /* The guest's view of its CR4 is unblemished. */
3927 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, uGuestCR4);
3928 AssertRCReturn(rc, rc);
3929 Log4Func(("uShadowCR4=%#RX32\n", uGuestCR4));
3930
3931 /*
3932 * Setup VT-x's view of the guest CR4.
3933 *
3934 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3935 * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3936 * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3937 *
3938 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3939 */
3940 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3941 {
3942 Assert(pVM->hm.s.vmx.pRealModeTSS);
3943 Assert(PDMVmmDevHeapIsEnabled(pVM));
3944 uGuestCR4 &= ~X86_CR4_VME;
3945 }
3946
3947 if (pVM->hm.s.fNestedPaging)
3948 {
3949 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3950 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3951 {
3952 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3953 uGuestCR4 |= X86_CR4_PSE;
3954 /* Our identity mapping is a 32-bit page directory. */
3955 uGuestCR4 &= ~X86_CR4_PAE;
3956 }
3957 /* else use guest CR4.*/
3958 }
3959 else
3960 {
3961 /*
3962 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3963 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3964 */
3965 switch (pVCpu->hm.s.enmShadowMode)
3966 {
3967 case PGMMODE_REAL: /* Real-mode. */
3968 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3969 case PGMMODE_32_BIT: /* 32-bit paging. */
3970 {
3971 uGuestCR4 &= ~X86_CR4_PAE;
3972 break;
3973 }
3974
3975 case PGMMODE_PAE: /* PAE paging. */
3976 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3977 {
3978 uGuestCR4 |= X86_CR4_PAE;
3979 break;
3980 }
3981
3982 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3983 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3984#ifdef VBOX_ENABLE_64_BITS_GUESTS
3985 break;
3986#endif
3987 default:
3988 AssertFailed();
3989 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3990 }
3991 }
3992
3993 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3994 uint64_t fSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3995 uint64_t fZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3996 uGuestCR4 |= fSetCR4;
3997 uGuestCR4 &= fZapCR4;
3998
3999 /* Write VT-x's view of the guest CR4 into the VMCS. */
4000 Log4Func(("uGuestCR4=%#RX32 (fSetCR4=%#RX32 fZapCR4=%#RX32)\n", uGuestCR4, fSetCR4, fZapCR4));
4001 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, uGuestCR4);
4002 AssertRCReturn(rc, rc);
4003
4004 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4005 uint32_t u32CR4Mask = X86_CR4_VME
4006 | X86_CR4_PAE
4007 | X86_CR4_PGE
4008 | X86_CR4_PSE
4009 | X86_CR4_VMXE;
4010 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4011 u32CR4Mask |= X86_CR4_OSXSAVE;
4012 if (pVM->cpum.ro.GuestFeatures.fPcid)
4013 u32CR4Mask |= X86_CR4_PCIDE;
4014 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4015 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4016 AssertRCReturn(rc, rc);
4017
4018 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4019 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4020
4021 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4022 }
4023 return rc;
4024}
4025
4026
4027/**
4028 * Exports the guest debug registers into the guest-state area in the VMCS.
4029 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4030 *
4031 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4032 *
4033 * @returns VBox status code.
4034 * @param pVCpu The cross context virtual CPU structure.
4035 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4036 * out-of-sync. Make sure to update the required fields
4037 * before using them.
4038 *
4039 * @remarks No-long-jump zone!!!
4040 */
4041static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4042{
4043 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4044
4045#ifdef VBOX_STRICT
4046 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4047 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4048 {
4049 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4050 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4051 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4052 }
4053#endif
4054
4055 bool fSteppingDB = false;
4056 bool fInterceptMovDRx = false;
4057 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4058 if (pVCpu->hm.s.fSingleInstruction)
4059 {
4060 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4061 PVM pVM = pVCpu->CTX_SUFF(pVM);
4062 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4063 {
4064 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4065 Assert(fSteppingDB == false);
4066 }
4067 else
4068 {
4069 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4070 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
4071 pVCpu->hm.s.fClearTrapFlag = true;
4072 fSteppingDB = true;
4073 }
4074 }
4075
4076 uint32_t uGuestDR7;
4077 if ( fSteppingDB
4078 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4079 {
4080 /*
4081 * Use the combined guest and host DRx values found in the hypervisor register set
4082 * because the debugger has breakpoints active or someone is single stepping on the
4083 * host side without a monitor trap flag.
4084 *
4085 * Note! DBGF expects a clean DR6 state before executing guest code.
4086 */
4087#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4088 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4089 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4090 {
4091 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4092 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4093 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4094 }
4095 else
4096#endif
4097 if (!CPUMIsHyperDebugStateActive(pVCpu))
4098 {
4099 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4100 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4101 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4102 }
4103
4104 /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4105 uGuestDR7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4106 pVCpu->hm.s.fUsingHyperDR7 = true;
4107 fInterceptMovDRx = true;
4108 }
4109 else
4110 {
4111 /*
4112 * If the guest has enabled debug registers, we need to load them prior to
4113 * executing guest code so they'll trigger at the right time.
4114 */
4115 if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
4116 {
4117#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4118 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4119 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4120 {
4121 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4122 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4123 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4124 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4125 }
4126 else
4127#endif
4128 if (!CPUMIsGuestDebugStateActive(pVCpu))
4129 {
4130 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4131 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4132 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4133 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4134 }
4135 Assert(!fInterceptMovDRx);
4136 }
4137 /*
4138 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4139 * must intercept #DB in order to maintain a correct DR6 guest value, and
4140 * because we need to intercept it to prevent nested #DBs from hanging the
4141 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4142 */
4143#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4144 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4145 && !CPUMIsGuestDebugStateActive(pVCpu))
4146#else
4147 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4148#endif
4149 {
4150 fInterceptMovDRx = true;
4151 }
4152
4153 /* Update DR7 with the actual guest value. */
4154 uGuestDR7 = pMixedCtx->dr[7];
4155 pVCpu->hm.s.fUsingHyperDR7 = false;
4156 }
4157
4158 if (fInterceptMovDRx)
4159 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4160 else
4161 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4162
4163 /*
4164 * Update the processor-based VM-execution controls for MOV-DRx intercepts and the monitor-trap flag.
4165 */
4166 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4167 {
4168 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4169 AssertRCReturn(rc2, rc2);
4170 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4171 }
4172
4173 /*
4174 * Update guest DR7.
4175 */
4176 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, uGuestDR7);
4177 AssertRCReturn(rc, rc);
4178
4179 return VINF_SUCCESS;
4180}
4181
4182
4183#ifdef VBOX_STRICT
4184/**
4185 * Strict function to validate segment registers.
4186 *
4187 * @remarks Will import guest CR0 on strict builds during validation of
4188 * segments.
4189 */
4190static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCCPUMCTX pCtx)
4191{
4192 /*
4193 * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4194 *
4195 * The reason we check for attribute value 0 in this function and not just the unusable bit is
4196 * because hmR0VmxWriteSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4197 * and doesn't change the guest-context value.
4198 */
4199 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4200 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4201 && ( !CPUMIsGuestInRealModeEx(pCtx)
4202 && !CPUMIsGuestInV86ModeEx(pCtx)))
4203 {
4204 /* Protected mode checks */
4205 /* CS */
4206 Assert(pCtx->cs.Attr.n.u1Present);
4207 Assert(!(pCtx->cs.Attr.u & 0xf00));
4208 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4209 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4210 || !(pCtx->cs.Attr.n.u1Granularity));
4211 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4212 || (pCtx->cs.Attr.n.u1Granularity));
4213 /* CS cannot be loaded with NULL in protected mode. */
4214 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4215 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4216 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4217 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4218 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4219 else
4220 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4221 /* SS */
4222 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4223 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4224 if ( !(pCtx->cr0 & X86_CR0_PE)
4225 || pCtx->cs.Attr.n.u4Type == 3)
4226 {
4227 Assert(!pCtx->ss.Attr.n.u2Dpl);
4228 }
4229 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4230 {
4231 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4232 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4233 Assert(pCtx->ss.Attr.n.u1Present);
4234 Assert(!(pCtx->ss.Attr.u & 0xf00));
4235 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4236 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4237 || !(pCtx->ss.Attr.n.u1Granularity));
4238 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4239 || (pCtx->ss.Attr.n.u1Granularity));
4240 }
4241 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4242 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4243 {
4244 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4245 Assert(pCtx->ds.Attr.n.u1Present);
4246 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4247 Assert(!(pCtx->ds.Attr.u & 0xf00));
4248 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4249 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4250 || !(pCtx->ds.Attr.n.u1Granularity));
4251 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4252 || (pCtx->ds.Attr.n.u1Granularity));
4253 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4254 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4255 }
4256 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4257 {
4258 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4259 Assert(pCtx->es.Attr.n.u1Present);
4260 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4261 Assert(!(pCtx->es.Attr.u & 0xf00));
4262 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4263 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4264 || !(pCtx->es.Attr.n.u1Granularity));
4265 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4266 || (pCtx->es.Attr.n.u1Granularity));
4267 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4268 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4269 }
4270 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4271 {
4272 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4273 Assert(pCtx->fs.Attr.n.u1Present);
4274 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4275 Assert(!(pCtx->fs.Attr.u & 0xf00));
4276 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4277 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4278 || !(pCtx->fs.Attr.n.u1Granularity));
4279 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4280 || (pCtx->fs.Attr.n.u1Granularity));
4281 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4282 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4283 }
4284 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4285 {
4286 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4287 Assert(pCtx->gs.Attr.n.u1Present);
4288 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4289 Assert(!(pCtx->gs.Attr.u & 0xf00));
4290 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4291 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4292 || !(pCtx->gs.Attr.n.u1Granularity));
4293 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4294 || (pCtx->gs.Attr.n.u1Granularity));
4295 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4296 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4297 }
4298 /* 64-bit capable CPUs. */
4299# if HC_ARCH_BITS == 64
4300 Assert(!(pCtx->cs.u64Base >> 32));
4301 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4302 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4303 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4304# endif
4305 }
4306 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4307 || ( CPUMIsGuestInRealModeEx(pCtx)
4308 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4309 {
4310 /* Real and v86 mode checks. */
4311 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4312 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4313 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4314 {
4315 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4316 }
4317 else
4318 {
4319 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4320 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4321 }
4322
4323 /* CS */
4324 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4325 Assert(pCtx->cs.u32Limit == 0xffff);
4326 Assert(u32CSAttr == 0xf3);
4327 /* SS */
4328 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4329 Assert(pCtx->ss.u32Limit == 0xffff);
4330 Assert(u32SSAttr == 0xf3);
4331 /* DS */
4332 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4333 Assert(pCtx->ds.u32Limit == 0xffff);
4334 Assert(u32DSAttr == 0xf3);
4335 /* ES */
4336 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4337 Assert(pCtx->es.u32Limit == 0xffff);
4338 Assert(u32ESAttr == 0xf3);
4339 /* FS */
4340 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4341 Assert(pCtx->fs.u32Limit == 0xffff);
4342 Assert(u32FSAttr == 0xf3);
4343 /* GS */
4344 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4345 Assert(pCtx->gs.u32Limit == 0xffff);
4346 Assert(u32GSAttr == 0xf3);
4347 /* 64-bit capable CPUs. */
4348# if HC_ARCH_BITS == 64
4349 Assert(!(pCtx->cs.u64Base >> 32));
4350 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4351 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4352 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4353# endif
4354 }
4355}
4356#endif /* VBOX_STRICT */
4357
4358
4359/**
4360 * Writes a guest segment register into the guest-state area in the VMCS.
4361 *
4362 * @returns VBox status code.
4363 * @param pVCpu The cross context virtual CPU structure.
4364 * @param idxSel Index of the selector in the VMCS.
4365 * @param idxLimit Index of the segment limit in the VMCS.
4366 * @param idxBase Index of the segment base in the VMCS.
4367 * @param idxAccess Index of the access rights of the segment in the VMCS.
4368 * @param pSelReg Pointer to the segment selector.
4369 *
4370 * @remarks No-long-jump zone!!!
4371 */
4372static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4373 uint32_t idxAccess, PCCPUMSELREG pSelReg)
4374{
4375 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4376 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4377 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4378 AssertRCReturn(rc, rc);
4379
4380 uint32_t u32Access = pSelReg->Attr.u;
4381 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4382 {
4383 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4384 u32Access = 0xf3;
4385 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4386 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4387 }
4388 else
4389 {
4390 /*
4391 * The way to differentiate between whether this is really a null selector or was just
4392 * a selector loaded with 0 in real-mode is using the segment attributes. A selector
4393 * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4394 * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4395 * NULL selectors loaded in protected-mode have their attribute as 0.
4396 */
4397 if (!u32Access)
4398 u32Access = X86DESCATTR_UNUSABLE;
4399 }
4400
4401 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4402 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4403 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4404
4405 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4406 AssertRCReturn(rc, rc);
4407 return rc;
4408}
4409
4410
4411/**
4412 * Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4413 * into the guest-state area in the VMCS.
4414 *
4415 * @returns VBox status code.
4416 * @param pVCpu The cross context virtual CPU structure.
4417 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4418 * out-of-sync. Make sure to update the required fields
4419 * before using them.
4420 *
4421 * @remarks Will import guest CR0 on strict builds during validation of
4422 * segments.
4423 * @remarks No-long-jump zone!!!
4424 */
4425static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4426{
4427 int rc = VERR_INTERNAL_ERROR_5;
4428 PVM pVM = pVCpu->CTX_SUFF(pVM);
4429
4430 /*
4431 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4432 */
4433 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4434 {
4435 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4436 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4437 {
4438 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4439 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4440 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4441 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4442 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4443 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4444 }
4445
4446#ifdef VBOX_WITH_REM
4447 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4448 {
4449 Assert(pVM->hm.s.vmx.pRealModeTSS);
4450 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4451 if ( pVCpu->hm.s.vmx.fWasInRealMode
4452 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4453 {
4454 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4455 in real-mode (e.g. OpenBSD 4.0) */
4456 REMFlushTBs(pVM);
4457 Log4Func(("Switch to protected mode detected!\n"));
4458 pVCpu->hm.s.vmx.fWasInRealMode = false;
4459 }
4460 }
4461#endif
4462 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4463 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4464 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4465 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4466 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4467 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4468 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4469 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4470 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4471 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4472 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4473 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4474 AssertRCReturn(rc, rc);
4475
4476#ifdef VBOX_STRICT
4477 /* Validate. */
4478 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4479#endif
4480
4481 /* Update the exit history entry with the correct CS.BASE + RIP. */
4482 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4483 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
4484
4485 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SREG_MASK);
4486 Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4487 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4488 }
4489
4490 /*
4491 * Guest TR.
4492 */
4493 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4494 {
4495 /*
4496 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4497 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4498 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4499 */
4500 uint16_t u16Sel = 0;
4501 uint32_t u32Limit = 0;
4502 uint64_t u64Base = 0;
4503 uint32_t u32AccessRights = 0;
4504
4505 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4506 {
4507 u16Sel = pMixedCtx->tr.Sel;
4508 u32Limit = pMixedCtx->tr.u32Limit;
4509 u64Base = pMixedCtx->tr.u64Base;
4510 u32AccessRights = pMixedCtx->tr.Attr.u;
4511 }
4512 else
4513 {
4514 Assert(pVM->hm.s.vmx.pRealModeTSS);
4515 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4516
4517 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4518 RTGCPHYS GCPhys;
4519 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4520 AssertRCReturn(rc, rc);
4521
4522 X86DESCATTR DescAttr;
4523 DescAttr.u = 0;
4524 DescAttr.n.u1Present = 1;
4525 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4526
4527 u16Sel = 0;
4528 u32Limit = HM_VTX_TSS_SIZE;
4529 u64Base = GCPhys; /* in real-mode phys = virt. */
4530 u32AccessRights = DescAttr.u;
4531 }
4532
4533 /* Validate. */
4534 Assert(!(u16Sel & RT_BIT(2)));
4535 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4536 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4537 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4538 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4539 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4540 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4541 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4542 Assert( (u32Limit & 0xfff) == 0xfff
4543 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4544 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4545 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4546
4547 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4548 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4549 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4550 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4551 AssertRCReturn(rc, rc);
4552
4553 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4554 Log4Func(("TR base=%#RX64\n", pMixedCtx->tr.u64Base));
4555 }
4556
4557 /*
4558 * Guest GDTR.
4559 */
4560 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4561 {
4562 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4563 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4564 AssertRCReturn(rc, rc);
4565
4566 /* Validate. */
4567 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4568
4569 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4570 Log4Func(("GDTR base=%#RX64\n", pMixedCtx->gdtr.pGdt));
4571 }
4572
4573 /*
4574 * Guest LDTR.
4575 */
4576 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4577 {
4578 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4579 uint32_t u32Access = 0;
4580 if (!pMixedCtx->ldtr.Attr.u)
4581 u32Access = X86DESCATTR_UNUSABLE;
4582 else
4583 u32Access = pMixedCtx->ldtr.Attr.u;
4584
4585 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4586 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4587 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4588 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4589 AssertRCReturn(rc, rc);
4590
4591 /* Validate. */
4592 if (!(u32Access & X86DESCATTR_UNUSABLE))
4593 {
4594 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4595 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4596 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4597 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4598 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4599 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4600 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4601 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4602 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4603 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4604 }
4605
4606 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4607 Log4Func(("LDTR base=%#RX64\n", pMixedCtx->ldtr.u64Base));
4608 }
4609
4610 /*
4611 * Guest IDTR.
4612 */
4613 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4614 {
4615 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4616 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4617 AssertRCReturn(rc, rc);
4618
4619 /* Validate. */
4620 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4621
4622 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4623 Log4Func(("IDTR base=%#RX64\n", pMixedCtx->idtr.pIdt));
4624 }
4625
4626 return VINF_SUCCESS;
4627}
4628
4629
4630/**
4631 * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4632 * areas.
4633 *
4634 * These MSRs will automatically be loaded to the host CPU on every successful
4635 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4636 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4637 * -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4638 *
4639 * Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4640 *
4641 * @returns VBox status code.
4642 * @param pVCpu The cross context virtual CPU structure.
4643 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4644 * out-of-sync. Make sure to update the required fields
4645 * before using them.
4646 *
4647 * @remarks No-long-jump zone!!!
4648 */
4649static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4650{
4651 AssertPtr(pVCpu);
4652 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4653
4654 /*
4655 * MSRs that we use the auto-load/store MSR area in the VMCS.
4656 * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4657 */
4658 PVM pVM = pVCpu->CTX_SUFF(pVM);
4659 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4660 {
4661 if (pVM->hm.s.fAllow64BitGuests)
4662 {
4663#if HC_ARCH_BITS == 32
4664 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4665 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4666 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4667 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4668 AssertRCReturn(rc, rc);
4669# ifdef LOG_ENABLED
4670 PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4671 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4672 Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4673# endif
4674#endif
4675 }
4676 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4677 }
4678
4679 /*
4680 * Guest Sysenter MSRs.
4681 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4682 * VM-exits on WRMSRs for these MSRs.
4683 */
4684 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4685 {
4686 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4687 {
4688 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs);
4689 AssertRCReturn(rc, rc);
4690 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4691 }
4692
4693 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4694 {
4695 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip);
4696 AssertRCReturn(rc, rc);
4697 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4698 }
4699
4700 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4701 {
4702 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp);
4703 AssertRCReturn(rc, rc);
4704 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4705 }
4706 }
4707
4708 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4709 {
4710 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4711 {
4712 /*
4713 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4714 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4715 */
4716 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4717 {
4718 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4719 AssertRCReturn(rc,rc);
4720 Log4Func(("EFER=%#RX64\n", pMixedCtx->msrEFER));
4721 }
4722 else
4723 {
4724 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4725 NULL /* pfAddedAndUpdated */);
4726 AssertRCReturn(rc, rc);
4727
4728 /* We need to intercept reads too, see @bugref{7386#c16}. */
4729 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4730 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4731 Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pMixedCtx->msrEFER,
4732 pVCpu->hm.s.vmx.cMsrs));
4733 }
4734 }
4735 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4736 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4737 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4738 }
4739
4740 return VINF_SUCCESS;
4741}
4742
4743
4744#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4745/**
4746 * Check if guest state allows safe use of 32-bit switcher again.
4747 *
4748 * Segment bases and protected mode structures must be 32-bit addressable
4749 * because the 32-bit switcher will ignore high dword when writing these VMCS
4750 * fields. See @bugref{8432} for details.
4751 *
4752 * @returns true if safe, false if must continue to use the 64-bit switcher.
4753 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4754 * out-of-sync. Make sure to update the required fields
4755 * before using them.
4756 *
4757 * @remarks No-long-jump zone!!!
4758 */
4759static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pMixedCtx)
4760{
4761 if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4762 if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4763 if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4764 if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4765 if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4766 if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4767 if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4768 if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4769 if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4770 if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4771
4772 /* All good, bases are 32-bit. */
4773 return true;
4774}
4775#endif
4776
4777
4778/**
4779 * Selects up the appropriate function to run guest code.
4780 *
4781 * @returns VBox status code.
4782 * @param pVCpu The cross context virtual CPU structure.
4783 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4784 * out-of-sync. Make sure to update the required fields
4785 * before using them.
4786 *
4787 * @remarks No-long-jump zone!!!
4788 */
4789static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4790{
4791 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4792 {
4793#ifndef VBOX_ENABLE_64_BITS_GUESTS
4794 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4795#endif
4796 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4797#if HC_ARCH_BITS == 32
4798 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4799 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4800 {
4801#ifdef VBOX_STRICT
4802 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4803 {
4804 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4805 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4806 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4807 | HM_CHANGED_VMX_ENTRY_CTLS
4808 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4809 }
4810#endif
4811 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4812
4813 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4814 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4815 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4816 Log4Func(("Selected 64-bit switcher\n"));
4817 }
4818#else
4819 /* 64-bit host. */
4820 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4821#endif
4822 }
4823 else
4824 {
4825 /* Guest is not in long mode, use the 32-bit handler. */
4826#if HC_ARCH_BITS == 32
4827 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4828 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4829 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4830 {
4831# ifdef VBOX_STRICT
4832 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4833 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4834 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4835 | HM_CHANGED_VMX_ENTRY_CTLS
4836 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4837# endif
4838 }
4839# ifdef VBOX_ENABLE_64_BITS_GUESTS
4840 /*
4841 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4842 * design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4843 * switcher flag because now we know the guest is in a sane state where it's safe
4844 * to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4845 * the much faster 32-bit switcher again.
4846 */
4847 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4848 {
4849 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4850 Log4Func(("Selected 32-bit switcher\n"));
4851 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4852 }
4853 else
4854 {
4855 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4856 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4857 || hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
4858 {
4859 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4860 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4861 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4862 | HM_CHANGED_VMX_ENTRY_CTLS
4863 | HM_CHANGED_VMX_EXIT_CTLS
4864 | HM_CHANGED_HOST_CONTEXT);
4865 Log4Func(("Selected 32-bit switcher (safe)\n"));
4866 }
4867 }
4868# else
4869 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4870# endif
4871#else
4872 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4873#endif
4874 }
4875 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4876 return VINF_SUCCESS;
4877}
4878
4879
4880/**
4881 * Wrapper for running the guest code in VT-x.
4882 *
4883 * @returns VBox status code, no informational status codes.
4884 * @param pVM The cross context VM structure.
4885 * @param pVCpu The cross context virtual CPU structure.
4886 * @param pCtx Pointer to the guest-CPU context.
4887 *
4888 * @remarks No-long-jump zone!!!
4889 */
4890DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4891{
4892 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4893 pCtx->fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
4894
4895 /*
4896 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4897 * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4898 * callee-saved and thus the need for this XMM wrapper.
4899 *
4900 * See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4901 */
4902 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4903 /** @todo Add stats for resume vs launch. */
4904#ifdef VBOX_WITH_KERNEL_USING_XMM
4905 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4906#else
4907 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4908#endif
4909 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4910 return rc;
4911}
4912
4913
4914/**
4915 * Reports world-switch error and dumps some useful debug info.
4916 *
4917 * @param pVM The cross context VM structure.
4918 * @param pVCpu The cross context virtual CPU structure.
4919 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4920 * @param pCtx Pointer to the guest-CPU context.
4921 * @param pVmxTransient Pointer to the VMX transient structure (only
4922 * exitReason updated).
4923 */
4924static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4925{
4926 Assert(pVM);
4927 Assert(pVCpu);
4928 Assert(pCtx);
4929 Assert(pVmxTransient);
4930 HMVMX_ASSERT_PREEMPT_SAFE();
4931
4932 Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
4933 switch (rcVMRun)
4934 {
4935 case VERR_VMX_INVALID_VMXON_PTR:
4936 AssertFailed();
4937 break;
4938 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4939 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4940 {
4941 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4942 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4943 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4944 AssertRC(rc);
4945
4946 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4947 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4948 Cannot do it here as we may have been long preempted. */
4949
4950#ifdef VBOX_STRICT
4951 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4952 pVmxTransient->uExitReason));
4953 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4954 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4955 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4956 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4957 else
4958 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4959 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4960 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4961
4962 /* VMX control bits. */
4963 uint32_t u32Val;
4964 uint64_t u64Val;
4965 RTHCUINTREG uHCReg;
4966 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4967 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4968 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4969 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4970 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
4971 {
4972 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4973 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4974 }
4975 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4976 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4977 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4978 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4979 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4980 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4981 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4982 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4983 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4984 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4985 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4986 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4987 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4988 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4989 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4990 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4991 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4992 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4993 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4994 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4995 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4996 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4997 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4998 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4999 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5000 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5001 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5002 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5003 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5004 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5005 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5006 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5007 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5008 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5009 if (pVM->hm.s.fNestedPaging)
5010 {
5011 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5012 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5013 }
5014
5015 /* Guest bits. */
5016 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5017 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5018 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5019 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5020 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5021 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5022 if (pVM->hm.s.vmx.fVpid)
5023 {
5024 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5025 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5026 }
5027
5028 /* Host bits. */
5029 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5030 Log4(("Host CR0 %#RHr\n", uHCReg));
5031 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5032 Log4(("Host CR3 %#RHr\n", uHCReg));
5033 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5034 Log4(("Host CR4 %#RHr\n", uHCReg));
5035
5036 RTGDTR HostGdtr;
5037 PCX86DESCHC pDesc;
5038 ASMGetGDTR(&HostGdtr);
5039 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5040 Log4(("Host CS %#08x\n", u32Val));
5041 if (u32Val < HostGdtr.cbGdt)
5042 {
5043 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5044 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5045 }
5046
5047 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5048 Log4(("Host DS %#08x\n", u32Val));
5049 if (u32Val < HostGdtr.cbGdt)
5050 {
5051 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5052 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5053 }
5054
5055 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5056 Log4(("Host ES %#08x\n", u32Val));
5057 if (u32Val < HostGdtr.cbGdt)
5058 {
5059 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5060 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5061 }
5062
5063 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5064 Log4(("Host FS %#08x\n", u32Val));
5065 if (u32Val < HostGdtr.cbGdt)
5066 {
5067 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5068 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5069 }
5070
5071 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5072 Log4(("Host GS %#08x\n", u32Val));
5073 if (u32Val < HostGdtr.cbGdt)
5074 {
5075 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5076 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5077 }
5078
5079 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5080 Log4(("Host SS %#08x\n", u32Val));
5081 if (u32Val < HostGdtr.cbGdt)
5082 {
5083 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5084 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5085 }
5086
5087 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5088 Log4(("Host TR %#08x\n", u32Val));
5089 if (u32Val < HostGdtr.cbGdt)
5090 {
5091 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5092 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5093 }
5094
5095 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5096 Log4(("Host TR Base %#RHv\n", uHCReg));
5097 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5098 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5099 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5100 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5101 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5102 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5103 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5104 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5105 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5106 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5107 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5108 Log4(("Host RSP %#RHv\n", uHCReg));
5109 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5110 Log4(("Host RIP %#RHv\n", uHCReg));
5111# if HC_ARCH_BITS == 64
5112 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5113 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5114 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5115 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5116 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5117 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5118# endif
5119#endif /* VBOX_STRICT */
5120 break;
5121 }
5122
5123 default:
5124 /* Impossible */
5125 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5126 break;
5127 }
5128 NOREF(pVM); NOREF(pCtx);
5129}
5130
5131
5132#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5133#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5134# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5135#endif
5136#ifdef VBOX_STRICT
5137static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5138{
5139 switch (idxField)
5140 {
5141 case VMX_VMCS_GUEST_RIP:
5142 case VMX_VMCS_GUEST_RSP:
5143 case VMX_VMCS_GUEST_SYSENTER_EIP:
5144 case VMX_VMCS_GUEST_SYSENTER_ESP:
5145 case VMX_VMCS_GUEST_GDTR_BASE:
5146 case VMX_VMCS_GUEST_IDTR_BASE:
5147 case VMX_VMCS_GUEST_CS_BASE:
5148 case VMX_VMCS_GUEST_DS_BASE:
5149 case VMX_VMCS_GUEST_ES_BASE:
5150 case VMX_VMCS_GUEST_FS_BASE:
5151 case VMX_VMCS_GUEST_GS_BASE:
5152 case VMX_VMCS_GUEST_SS_BASE:
5153 case VMX_VMCS_GUEST_LDTR_BASE:
5154 case VMX_VMCS_GUEST_TR_BASE:
5155 case VMX_VMCS_GUEST_CR3:
5156 return true;
5157 }
5158 return false;
5159}
5160
5161static bool hmR0VmxIsValidReadField(uint32_t idxField)
5162{
5163 switch (idxField)
5164 {
5165 /* Read-only fields. */
5166 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5167 return true;
5168 }
5169 /* Remaining readable fields should also be writable. */
5170 return hmR0VmxIsValidWriteField(idxField);
5171}
5172#endif /* VBOX_STRICT */
5173
5174
5175/**
5176 * Executes the specified handler in 64-bit mode.
5177 *
5178 * @returns VBox status code (no informational status codes).
5179 * @param pVCpu The cross context virtual CPU structure.
5180 * @param enmOp The operation to perform.
5181 * @param cParams Number of parameters.
5182 * @param paParam Array of 32-bit parameters.
5183 */
5184VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp,
5185 uint32_t cParams, uint32_t *paParam)
5186{
5187 PVM pVM = pVCpu->CTX_SUFF(pVM);
5188 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5189 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5190 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5191 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5192
5193#ifdef VBOX_STRICT
5194 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5195 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5196
5197 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5198 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5199#endif
5200
5201 /* Disable interrupts. */
5202 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5203
5204#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5205 RTCPUID idHostCpu = RTMpCpuId();
5206 CPUMR0SetLApic(pVCpu, idHostCpu);
5207#endif
5208
5209 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5210 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5211
5212 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5213 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5214 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5215
5216 /* Leave VMX Root Mode. */
5217 VMXDisable();
5218
5219 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5220
5221 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5222 CPUMSetHyperEIP(pVCpu, enmOp);
5223 for (int i = (int)cParams - 1; i >= 0; i--)
5224 CPUMPushHyper(pVCpu, paParam[i]);
5225
5226 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5227
5228 /* Call the switcher. */
5229 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5230 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5231
5232 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5233 /* Make sure the VMX instructions don't cause #UD faults. */
5234 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5235
5236 /* Re-enter VMX Root Mode */
5237 int rc2 = VMXEnable(HCPhysCpuPage);
5238 if (RT_FAILURE(rc2))
5239 {
5240 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5241 ASMSetFlags(fOldEFlags);
5242 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5243 return rc2;
5244 }
5245
5246 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5247 AssertRC(rc2);
5248 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5249 Assert(!(ASMGetFlags() & X86_EFL_IF));
5250 ASMSetFlags(fOldEFlags);
5251 return rc;
5252}
5253
5254
5255/**
5256 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5257 * supporting 64-bit guests.
5258 *
5259 * @returns VBox status code.
5260 * @param fResume Whether to VMLAUNCH or VMRESUME.
5261 * @param pCtx Pointer to the guest-CPU context.
5262 * @param pCache Pointer to the VMCS cache.
5263 * @param pVM The cross context VM structure.
5264 * @param pVCpu The cross context virtual CPU structure.
5265 */
5266DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5267{
5268 NOREF(fResume);
5269
5270 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5271 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5272
5273#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5274 pCache->uPos = 1;
5275 pCache->interPD = PGMGetInterPaeCR3(pVM);
5276 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5277#endif
5278
5279#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5280 pCache->TestIn.HCPhysCpuPage = 0;
5281 pCache->TestIn.HCPhysVmcs = 0;
5282 pCache->TestIn.pCache = 0;
5283 pCache->TestOut.HCPhysVmcs = 0;
5284 pCache->TestOut.pCache = 0;
5285 pCache->TestOut.pCtx = 0;
5286 pCache->TestOut.eflags = 0;
5287#else
5288 NOREF(pCache);
5289#endif
5290
5291 uint32_t aParam[10];
5292 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5293 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5294 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5295 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5296 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5297 aParam[5] = 0;
5298 aParam[6] = VM_RC_ADDR(pVM, pVM);
5299 aParam[7] = 0;
5300 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5301 aParam[9] = 0;
5302
5303#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5304 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5305 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5306#endif
5307 int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5308
5309#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5310 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5311 Assert(pCtx->dr[4] == 10);
5312 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5313#endif
5314
5315#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5316 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5317 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5318 pVCpu->hm.s.vmx.HCPhysVmcs));
5319 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5320 pCache->TestOut.HCPhysVmcs));
5321 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5322 pCache->TestOut.pCache));
5323 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5324 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5325 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5326 pCache->TestOut.pCtx));
5327 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5328#endif
5329 NOREF(pCtx);
5330 return rc;
5331}
5332
5333
5334/**
5335 * Initialize the VMCS-Read cache.
5336 *
5337 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5338 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5339 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5340 * (those that have a 32-bit FULL & HIGH part).
5341 *
5342 * @returns VBox status code.
5343 * @param pVM The cross context VM structure.
5344 * @param pVCpu The cross context virtual CPU structure.
5345 */
5346static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5347{
5348#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5349{ \
5350 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5351 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5352 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5353 ++cReadFields; \
5354}
5355
5356 AssertPtr(pVM);
5357 AssertPtr(pVCpu);
5358 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5359 uint32_t cReadFields = 0;
5360
5361 /*
5362 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5363 * and serve to indicate exceptions to the rules.
5364 */
5365
5366 /* Guest-natural selector base fields. */
5367#if 0
5368 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5369 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5370 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5371#endif
5372 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5373 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5374 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5375 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5376 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5377 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5378 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5379 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5380 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5381 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5382 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5383 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5384#if 0
5385 /* Unused natural width guest-state fields. */
5386 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5387 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5388#endif
5389 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5390 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5391
5392 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5393 these 64-bit fields (using "FULL" and "HIGH" fields). */
5394#if 0
5395 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5396 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5397 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5398 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5399 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5400 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5401 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5402 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5403 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5404#endif
5405
5406 /* Natural width guest-state fields. */
5407 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5408#if 0
5409 /* Currently unused field. */
5410 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5411#endif
5412
5413 if (pVM->hm.s.fNestedPaging)
5414 {
5415 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5416 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5417 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5418 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5419 }
5420 else
5421 {
5422 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5423 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5424 }
5425
5426#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5427 return VINF_SUCCESS;
5428}
5429
5430
5431/**
5432 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5433 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5434 * darwin, running 64-bit guests).
5435 *
5436 * @returns VBox status code.
5437 * @param pVCpu The cross context virtual CPU structure.
5438 * @param idxField The VMCS field encoding.
5439 * @param u64Val 16, 32 or 64-bit value.
5440 */
5441VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5442{
5443 int rc;
5444 switch (idxField)
5445 {
5446 /*
5447 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5448 */
5449 /* 64-bit Control fields. */
5450 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5451 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5452 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5453 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5454 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5455 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5456 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5457 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5458 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5459 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5460 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5461 case VMX_VMCS64_CTRL_EPTP_FULL:
5462 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5463 /* 64-bit Guest-state fields. */
5464 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5465 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5466 case VMX_VMCS64_GUEST_PAT_FULL:
5467 case VMX_VMCS64_GUEST_EFER_FULL:
5468 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5469 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5470 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5471 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5472 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5473 /* 64-bit Host-state fields. */
5474 case VMX_VMCS64_HOST_PAT_FULL:
5475 case VMX_VMCS64_HOST_EFER_FULL:
5476 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5477 {
5478 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5479 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5480 break;
5481 }
5482
5483 /*
5484 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5485 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5486 */
5487 /* Natural-width Guest-state fields. */
5488 case VMX_VMCS_GUEST_CR3:
5489 case VMX_VMCS_GUEST_ES_BASE:
5490 case VMX_VMCS_GUEST_CS_BASE:
5491 case VMX_VMCS_GUEST_SS_BASE:
5492 case VMX_VMCS_GUEST_DS_BASE:
5493 case VMX_VMCS_GUEST_FS_BASE:
5494 case VMX_VMCS_GUEST_GS_BASE:
5495 case VMX_VMCS_GUEST_LDTR_BASE:
5496 case VMX_VMCS_GUEST_TR_BASE:
5497 case VMX_VMCS_GUEST_GDTR_BASE:
5498 case VMX_VMCS_GUEST_IDTR_BASE:
5499 case VMX_VMCS_GUEST_RSP:
5500 case VMX_VMCS_GUEST_RIP:
5501 case VMX_VMCS_GUEST_SYSENTER_ESP:
5502 case VMX_VMCS_GUEST_SYSENTER_EIP:
5503 {
5504 if (!(RT_HI_U32(u64Val)))
5505 {
5506 /* If this field is 64-bit, VT-x will zero out the top bits. */
5507 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5508 }
5509 else
5510 {
5511 /* Assert that only the 32->64 switcher case should ever come here. */
5512 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5513 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5514 }
5515 break;
5516 }
5517
5518 default:
5519 {
5520 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5521 rc = VERR_INVALID_PARAMETER;
5522 break;
5523 }
5524 }
5525 AssertRCReturn(rc, rc);
5526 return rc;
5527}
5528
5529
5530/**
5531 * Queue up a VMWRITE by using the VMCS write cache.
5532 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5533 *
5534 * @param pVCpu The cross context virtual CPU structure.
5535 * @param idxField The VMCS field encoding.
5536 * @param u64Val 16, 32 or 64-bit value.
5537 */
5538VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5539{
5540 AssertPtr(pVCpu);
5541 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5542
5543 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5544 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5545
5546 /* Make sure there are no duplicates. */
5547 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5548 {
5549 if (pCache->Write.aField[i] == idxField)
5550 {
5551 pCache->Write.aFieldVal[i] = u64Val;
5552 return VINF_SUCCESS;
5553 }
5554 }
5555
5556 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5557 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5558 pCache->Write.cValidEntries++;
5559 return VINF_SUCCESS;
5560}
5561#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5562
5563
5564/**
5565 * Sets up the usage of TSC-offsetting and updates the VMCS.
5566 *
5567 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5568 * VMX preemption timer.
5569 *
5570 * @returns VBox status code.
5571 * @param pVM The cross context VM structure.
5572 * @param pVCpu The cross context virtual CPU structure.
5573 *
5574 * @remarks No-long-jump zone!!!
5575 */
5576static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5577{
5578 int rc;
5579 bool fOffsettedTsc;
5580 bool fParavirtTsc;
5581 if (pVM->hm.s.vmx.fUsePreemptTimer)
5582 {
5583 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5584 &fOffsettedTsc, &fParavirtTsc);
5585
5586 /* Make sure the returned values have sane upper and lower boundaries. */
5587 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5588 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5589 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5590 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5591
5592 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5593 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5594 }
5595 else
5596 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5597
5598 /** @todo later optimize this to be done elsewhere and not before every
5599 * VM-entry. */
5600 if (fParavirtTsc)
5601 {
5602 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5603 information before every VM-entry, hence disable it for performance sake. */
5604#if 0
5605 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5606 AssertRC(rc);
5607#endif
5608 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5609 }
5610
5611 if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5612 {
5613 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5614 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5615
5616 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5617 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5618 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5619 }
5620 else
5621 {
5622 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5623 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5624 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5625 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5626 }
5627}
5628
5629
5630/**
5631 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5632 * VM-exit interruption info type.
5633 *
5634 * @returns The IEM exception flags.
5635 * @param uVector The event vector.
5636 * @param uVmxVectorType The VMX event type.
5637 *
5638 * @remarks This function currently only constructs flags required for
5639 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5640 * and CR2 aspects of an exception are not included).
5641 */
5642static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5643{
5644 uint32_t fIemXcptFlags;
5645 switch (uVmxVectorType)
5646 {
5647 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5648 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5649 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5650 break;
5651
5652 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5653 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5654 break;
5655
5656 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5657 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5658 break;
5659
5660 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5661 {
5662 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5663 if (uVector == X86_XCPT_BP)
5664 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5665 else if (uVector == X86_XCPT_OF)
5666 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5667 else
5668 {
5669 fIemXcptFlags = 0;
5670 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5671 }
5672 break;
5673 }
5674
5675 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5676 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5677 break;
5678
5679 default:
5680 fIemXcptFlags = 0;
5681 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5682 break;
5683 }
5684 return fIemXcptFlags;
5685}
5686
5687
5688/**
5689 * Sets an event as a pending event to be injected into the guest.
5690 *
5691 * @param pVCpu The cross context virtual CPU structure.
5692 * @param u32IntInfo The VM-entry interruption-information field.
5693 * @param cbInstr The VM-entry instruction length in bytes (for software
5694 * interrupts, exceptions and privileged software
5695 * exceptions).
5696 * @param u32ErrCode The VM-entry exception error code.
5697 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5698 * page-fault.
5699 *
5700 * @remarks Statistics counter assumes this is a guest event being injected or
5701 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5702 * always incremented.
5703 */
5704DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5705 RTGCUINTPTR GCPtrFaultAddress)
5706{
5707 Assert(!pVCpu->hm.s.Event.fPending);
5708 pVCpu->hm.s.Event.fPending = true;
5709 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5710 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5711 pVCpu->hm.s.Event.cbInstr = cbInstr;
5712 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5713}
5714
5715
5716/**
5717 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5718 *
5719 * @param pVCpu The cross context virtual CPU structure.
5720 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5721 * out-of-sync. Make sure to update the required fields
5722 * before using them.
5723 */
5724DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5725{
5726 NOREF(pMixedCtx);
5727 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5728 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5729 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5730 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5731}
5732
5733
5734/**
5735 * Handle a condition that occurred while delivering an event through the guest
5736 * IDT.
5737 *
5738 * @returns Strict VBox status code (i.e. informational status codes too).
5739 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5740 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5741 * to continue execution of the guest which will delivery the \#DF.
5742 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5743 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5744 *
5745 * @param pVCpu The cross context virtual CPU structure.
5746 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5747 * out-of-sync. Make sure to update the required fields
5748 * before using them.
5749 * @param pVmxTransient Pointer to the VMX transient structure.
5750 *
5751 * @remarks No-long-jump zone!!!
5752 */
5753static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5754{
5755 uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5756
5757 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5758 rc2 |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5759 AssertRCReturn(rc2, rc2);
5760
5761 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5762 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5763 {
5764 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5765 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5766
5767 /*
5768 * If the event was a software interrupt (generated with INT n) or a software exception
5769 * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
5770 * can handle the VM-exit and continue guest execution which will re-execute the
5771 * instruction rather than re-injecting the exception, as that can cause premature
5772 * trips to ring-3 before injection and involve TRPM which currently has no way of
5773 * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
5774 * the problem).
5775 */
5776 IEMXCPTRAISE enmRaise;
5777 IEMXCPTRAISEINFO fRaiseInfo;
5778 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5779 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5780 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5781 {
5782 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5783 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5784 }
5785 else if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5786 {
5787 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5788 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5789 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5790 /** @todo Make AssertMsgReturn as just AssertMsg later. */
5791 AssertMsgReturn(uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT,
5792 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5793 uExitVectorType), VERR_VMX_IPE_5);
5794
5795 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5796
5797 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5798 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
5799 {
5800 pVmxTransient->fVectoringPF = true;
5801 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5802 }
5803 }
5804 else
5805 {
5806 /*
5807 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5808 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5809 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
5810 */
5811 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5812 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5813 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5814 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5815 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5816 }
5817
5818 /*
5819 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5820 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5821 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5822 * subsequent VM-entry would fail.
5823 *
5824 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5825 */
5826 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5827 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5828 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
5829 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
5830 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5831 {
5832 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5833 }
5834
5835 switch (enmRaise)
5836 {
5837 case IEMXCPTRAISE_CURRENT_XCPT:
5838 {
5839 Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
5840 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
5841 Assert(rcStrict == VINF_SUCCESS);
5842 break;
5843 }
5844
5845 case IEMXCPTRAISE_PREV_EVENT:
5846 {
5847 uint32_t u32ErrCode;
5848 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5849 {
5850 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5851 AssertRCReturn(rc2, rc2);
5852 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5853 }
5854 else
5855 u32ErrCode = 0;
5856
5857 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
5858 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5859 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5860 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5861
5862 Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
5863 pVCpu->hm.s.Event.u32ErrCode));
5864 Assert(rcStrict == VINF_SUCCESS);
5865 break;
5866 }
5867
5868 case IEMXCPTRAISE_REEXEC_INSTR:
5869 Assert(rcStrict == VINF_SUCCESS);
5870 break;
5871
5872 case IEMXCPTRAISE_DOUBLE_FAULT:
5873 {
5874 /*
5875 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
5876 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
5877 */
5878 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
5879 {
5880 pVmxTransient->fVectoringDoublePF = true;
5881 Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
5882 pMixedCtx->cr2));
5883 rcStrict = VINF_SUCCESS;
5884 }
5885 else
5886 {
5887 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5888 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5889 Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
5890 uIdtVector, uExitVector));
5891 rcStrict = VINF_HM_DOUBLE_FAULT;
5892 }
5893 break;
5894 }
5895
5896 case IEMXCPTRAISE_TRIPLE_FAULT:
5897 {
5898 Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
5899 rcStrict = VINF_EM_RESET;
5900 break;
5901 }
5902
5903 case IEMXCPTRAISE_CPU_HANG:
5904 {
5905 Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
5906 rcStrict = VERR_EM_GUEST_CPU_HANG;
5907 break;
5908 }
5909
5910 default:
5911 {
5912 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
5913 rcStrict = VERR_VMX_IPE_2;
5914 break;
5915 }
5916 }
5917 }
5918 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5919 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5920 && uExitVector != X86_XCPT_DF
5921 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5922 {
5923 /*
5924 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5925 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5926 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5927 */
5928 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5929 {
5930 Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
5931 VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5932 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5933 }
5934 }
5935
5936 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
5937 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
5938 return rcStrict;
5939}
5940
5941
5942/**
5943 * Imports a guest segment register from the current VMCS into
5944 * the guest-CPU context.
5945 *
5946 * @returns VBox status code.
5947 * @param pVCpu The cross context virtual CPU structure.
5948 * @param idxSel Index of the selector in the VMCS.
5949 * @param idxLimit Index of the segment limit in the VMCS.
5950 * @param idxBase Index of the segment base in the VMCS.
5951 * @param idxAccess Index of the access rights of the segment in the VMCS.
5952 * @param pSelReg Pointer to the segment selector.
5953 *
5954 * @remarks No-long-jump zone!!!
5955 *
5956 * @remarks Never call this function directly!!! Use the
5957 * HMVMX_IMPORT_SREG() macro as that takes care
5958 * of whether to read from the VMCS cache or not.
5959 */
5960static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5961 PCPUMSELREG pSelReg)
5962{
5963 NOREF(pVCpu);
5964
5965 uint32_t u32Sel;
5966 uint32_t u32Limit;
5967 uint32_t u32Attr;
5968 uint64_t u64Base;
5969 int rc = VMXReadVmcs32(idxSel, &u32Sel);
5970 rc |= VMXReadVmcs32(idxLimit, &u32Limit);
5971 rc |= VMXReadVmcs32(idxAccess, &u32Attr);
5972 rc |= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
5973 AssertRCReturn(rc, rc);
5974
5975 pSelReg->Sel = (uint16_t)u32Sel;
5976 pSelReg->ValidSel = (uint16_t)u32Sel;
5977 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5978 pSelReg->u32Limit = u32Limit;
5979 pSelReg->u64Base = u64Base;
5980 pSelReg->Attr.u = u32Attr;
5981
5982 /*
5983 * If VT-x marks the segment as unusable, most other bits remain undefined:
5984 * - For CS the L, D and G bits have meaning.
5985 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
5986 * - For the remaining data segments no bits are defined.
5987 *
5988 * The present bit and the unusable bit has been observed to be set at the
5989 * same time (the selector was supposed to be invalid as we started executing
5990 * a V8086 interrupt in ring-0).
5991 *
5992 * What should be important for the rest of the VBox code, is that the P bit is
5993 * cleared. Some of the other VBox code recognizes the unusable bit, but
5994 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
5995 * safe side here, we'll strip off P and other bits we don't care about. If
5996 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
5997 *
5998 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5999 */
6000 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6001 {
6002 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6003
6004 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6005 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6006 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6007
6008 Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
6009#ifdef DEBUG_bird
6010 AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
6011 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6012 idxSel, u32Sel, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6013#endif
6014 }
6015 return VINF_SUCCESS;
6016}
6017
6018
6019/**
6020 * Imports the guest RIP from the VMCS back into the guest-CPU context.
6021 *
6022 * @returns VBox status code.
6023 * @param pVCpu The cross context virtual CPU structure.
6024 *
6025 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6026 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6027 * instead!!!
6028 */
6029DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6030{
6031 uint64_t u64Val;
6032 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6033 if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6034 {
6035 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6036 if (RT_SUCCESS(rc))
6037 {
6038 pCtx->rip = u64Val;
6039 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6040 }
6041 return rc;
6042 }
6043 return VINF_SUCCESS;
6044}
6045
6046
6047/**
6048 * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6049 *
6050 * @returns VBox status code.
6051 * @param pVCpu The cross context virtual CPU structure.
6052 *
6053 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6054 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6055 * instead!!!
6056 */
6057DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6058{
6059 uint32_t u32Val;
6060 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6061 if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6062 {
6063 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6064 if (RT_SUCCESS(rc))
6065 {
6066 pCtx->eflags.u32 = u32Val;
6067
6068 /* Restore eflags for real-on-v86-mode hack. */
6069 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6070 {
6071 pCtx->eflags.Bits.u1VM = 0;
6072 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6073 }
6074 }
6075 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6076 return rc;
6077 }
6078 return VINF_SUCCESS;
6079}
6080
6081
6082/**
6083 * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6084 * context.
6085 *
6086 * @returns VBox status code.
6087 * @param pVCpu The cross context virtual CPU structure.
6088 *
6089 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6090 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6091 * instead!!!
6092 */
6093DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6094{
6095 uint32_t u32Val;
6096 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6097 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32Val);
6098 if (RT_SUCCESS(rc))
6099 {
6100 /*
6101 * We additionally have a requirement to import RIP, RFLAGS depending on whether we
6102 * might need them in hmR0VmxEvaluatePendingEvent().
6103 */
6104 if (!u32Val)
6105 {
6106 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6107 {
6108 rc = hmR0VmxImportGuestRip(pVCpu);
6109 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6110 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6111 }
6112
6113 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6114 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6115 }
6116 else
6117 {
6118 rc = hmR0VmxImportGuestRip(pVCpu);
6119 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6120
6121 if (u32Val & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6122 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6123 {
6124 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6125 }
6126 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6127 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6128
6129 if (u32Val & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6130 {
6131 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6132 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6133 }
6134 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6135 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6136 }
6137 }
6138 return rc;
6139}
6140
6141
6142/**
6143 * Worker for VMXR0ImportStateOnDemand.
6144 *
6145 * @returns VBox status code.
6146 * @param pVCpu The cross context virtual CPU structure.
6147 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6148 */
6149static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6150{
6151#define VMXLOCAL_BREAK_RC(a_rc) \
6152 if (RT_FAILURE(a_rc)) \
6153 break
6154
6155 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6156
6157 int rc = VINF_SUCCESS;
6158 PVM pVM = pVCpu->CTX_SUFF(pVM);
6159 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6160 uint64_t u64Val;
6161 uint32_t u32Val;
6162
6163 Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6164
6165 /*
6166 * We disable interrupts to make the updating of the state and in particular
6167 * the fExtrn modification atomic wrt to preemption hooks.
6168 */
6169 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6170
6171 fWhat &= pCtx->fExtrn;
6172 if (fWhat)
6173 {
6174 do
6175 {
6176 if (fWhat & CPUMCTX_EXTRN_RIP)
6177 {
6178 rc = hmR0VmxImportGuestRip(pVCpu);
6179 VMXLOCAL_BREAK_RC(rc);
6180 }
6181
6182 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6183 {
6184 rc = hmR0VmxImportGuestRFlags(pVCpu);
6185 VMXLOCAL_BREAK_RC(rc);
6186 }
6187
6188 if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6189 {
6190 rc = hmR0VmxImportGuestIntrState(pVCpu);
6191 VMXLOCAL_BREAK_RC(rc);
6192 }
6193
6194 if (fWhat & CPUMCTX_EXTRN_RSP)
6195 {
6196 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6197 VMXLOCAL_BREAK_RC(rc);
6198 pCtx->rsp = u64Val;
6199 }
6200
6201 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6202 {
6203 if (fWhat & CPUMCTX_EXTRN_CS)
6204 {
6205 rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6206 VMXLOCAL_BREAK_RC(rc);
6207 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6208 pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6209 }
6210 if (fWhat & CPUMCTX_EXTRN_SS)
6211 {
6212 rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6213 VMXLOCAL_BREAK_RC(rc);
6214 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6215 pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6216 }
6217 if (fWhat & CPUMCTX_EXTRN_DS)
6218 {
6219 rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6220 VMXLOCAL_BREAK_RC(rc);
6221 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6222 pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6223 }
6224 if (fWhat & CPUMCTX_EXTRN_ES)
6225 {
6226 rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6227 VMXLOCAL_BREAK_RC(rc);
6228 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6229 pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6230 }
6231 if (fWhat & CPUMCTX_EXTRN_FS)
6232 {
6233 rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6234 VMXLOCAL_BREAK_RC(rc);
6235 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6236 pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6237 }
6238 if (fWhat & CPUMCTX_EXTRN_GS)
6239 {
6240 rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6241 VMXLOCAL_BREAK_RC(rc);
6242 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6243 pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6244 }
6245 }
6246
6247 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6248 {
6249 if (fWhat & CPUMCTX_EXTRN_LDTR)
6250 {
6251 rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6252 VMXLOCAL_BREAK_RC(rc);
6253 }
6254
6255 if (fWhat & CPUMCTX_EXTRN_GDTR)
6256 {
6257 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6258 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6259 VMXLOCAL_BREAK_RC(rc);
6260 pCtx->gdtr.pGdt = u64Val;
6261 pCtx->gdtr.cbGdt = u32Val;
6262 }
6263
6264 /* Guest IDTR. */
6265 if (fWhat & CPUMCTX_EXTRN_IDTR)
6266 {
6267 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6268 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6269 VMXLOCAL_BREAK_RC(rc);
6270 pCtx->idtr.pIdt = u64Val;
6271 pCtx->idtr.cbIdt = u32Val;
6272 }
6273
6274 /* Guest TR. */
6275 if (fWhat & CPUMCTX_EXTRN_TR)
6276 {
6277 /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6278 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6279 {
6280 rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6281 VMXLOCAL_BREAK_RC(rc);
6282 }
6283 }
6284 }
6285
6286 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6287 {
6288 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6289 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6290 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6291 pCtx->SysEnter.cs = u32Val;
6292 VMXLOCAL_BREAK_RC(rc);
6293 }
6294
6295#if HC_ARCH_BITS == 64
6296 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6297 {
6298 if ( pVM->hm.s.fAllow64BitGuests
6299 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6300 pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6301 }
6302
6303 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6304 {
6305 if ( pVM->hm.s.fAllow64BitGuests
6306 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6307 {
6308 pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6309 pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6310 pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6311 }
6312 }
6313#endif
6314
6315 if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
6316#if HC_ARCH_BITS == 32
6317 || (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS))
6318#endif
6319 )
6320 {
6321 PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6322 uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6323 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6324 {
6325 switch (pMsr->u32Msr)
6326 {
6327#if HC_ARCH_BITS == 32
6328 case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6329 case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6330 case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6331 case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6332#endif
6333 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6334 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6335 default:
6336 {
6337 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6338 cMsrs));
6339 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6340 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6341 }
6342 }
6343 }
6344 }
6345
6346 if (fWhat & CPUMCTX_EXTRN_DR7)
6347 {
6348 if (!pVCpu->hm.s.fUsingHyperDR7)
6349 {
6350 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6351 rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6352 VMXLOCAL_BREAK_RC(rc);
6353 pCtx->dr[7] = u32Val;
6354 }
6355 }
6356
6357 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6358 {
6359 uint32_t u32Shadow;
6360 /* CR0 required for saving CR3 below, see CPUMIsGuestPagingEnabledEx(). */
6361 if (fWhat & CPUMCTX_EXTRN_CR0)
6362 {
6363 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6364 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6365 VMXLOCAL_BREAK_RC(rc);
6366 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32CR0Mask)
6367 | (u32Shadow & pVCpu->hm.s.vmx.u32CR0Mask);
6368 VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6369 CPUMSetGuestCR0(pVCpu, u32Val);
6370 VMMRZCallRing3Enable(pVCpu);
6371 }
6372
6373 /* CR4 required for saving CR3 below, see CPUMIsGuestInPAEModeEx(). */
6374 if (fWhat & CPUMCTX_EXTRN_CR4)
6375 {
6376 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6377 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6378 VMXLOCAL_BREAK_RC(rc);
6379 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32CR4Mask)
6380 | (u32Shadow & pVCpu->hm.s.vmx.u32CR4Mask);
6381 CPUMSetGuestCR4(pVCpu, u32Val);
6382 }
6383
6384 if (fWhat & CPUMCTX_EXTRN_CR3)
6385 {
6386 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6387 || ( pVM->hm.s.fNestedPaging
6388 && CPUMIsGuestPagingEnabledEx(pCtx))) /* PG bit changes are always intercepted, so it's up to date. */
6389 {
6390 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6391 if (pCtx->cr3 != u64Val)
6392 {
6393 CPUMSetGuestCR3(pVCpu, u64Val);
6394 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6395 }
6396
6397 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6398 if (CPUMIsGuestInPAEModeEx(pCtx))
6399 {
6400 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6401 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6402 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6403 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6404 VMXLOCAL_BREAK_RC(rc);
6405 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6406 }
6407 }
6408 }
6409 }
6410 } while (0);
6411
6412 if (RT_SUCCESS(rc))
6413 {
6414 /* Update fExtrn. */
6415 pCtx->fExtrn &= ~fWhat;
6416
6417 /* If everything has been imported, clear the HM keeper bit. */
6418 if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6419 {
6420 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6421 Assert(!pCtx->fExtrn);
6422 }
6423 }
6424 }
6425 else
6426 AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6427
6428 ASMSetFlags(fEFlags);
6429
6430 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6431
6432 /*
6433 * Honor any pending CR3 updates.
6434 *
6435 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6436 * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6437 * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6438 *
6439 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6440 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6441 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6442 * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6443 *
6444 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6445 */
6446 if (VMMRZCallRing3IsEnabled(pVCpu))
6447 {
6448 VMMR0LogFlushDisable(pVCpu);
6449
6450 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6451 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6452
6453 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6454 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6455
6456 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6457 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6458
6459 VMMR0LogFlushEnable(pVCpu);
6460 }
6461
6462 return VINF_SUCCESS;
6463#undef VMXLOCAL_BREAK_RC
6464}
6465
6466
6467/**
6468 * Saves the guest state from the VMCS into the guest-CPU context.
6469 *
6470 * @returns VBox status code.
6471 * @param pVCpu The cross context virtual CPU structure.
6472 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6473 */
6474VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6475{
6476 return hmR0VmxImportGuestState(pVCpu, fWhat);
6477}
6478
6479
6480/**
6481 * Check per-VM and per-VCPU force flag actions that require us to go back to
6482 * ring-3 for one reason or another.
6483 *
6484 * @returns Strict VBox status code (i.e. informational status codes too)
6485 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6486 * ring-3.
6487 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6488 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6489 * interrupts)
6490 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6491 * all EMTs to be in ring-3.
6492 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6493 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6494 * to the EM loop.
6495 *
6496 * @param pVM The cross context VM structure.
6497 * @param pVCpu The cross context virtual CPU structure.
6498 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6499 * out-of-sync. Make sure to update the required fields
6500 * before using them.
6501 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6502 */
6503static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6504{
6505 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6506
6507 /*
6508 * Anything pending? Should be more likely than not if we're doing a good job.
6509 */
6510 if ( !fStepping
6511 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6512 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6513 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6514 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6515 return VINF_SUCCESS;
6516
6517#if 0
6518 /* We need the control registers now, make sure the guest-CPU context is updated. */
6519 int rc3 = hmR0VmxImportGuestStatae(pVCpu, CPUMCTX_EXTRN_CR0);
6520 AssertRCReturn(rc3, rc3);
6521
6522 /** @todo r=ramshankar: VMCPU_FF_HM_UPDATE_CR3 and VMCPU_FF_HM_UPDATE_PAE_PDPES
6523 * are not part of VMCPU_FF_HP_R0_PRE_HM_MASK. Hence, the two if
6524 * statements below won't ever be entered. Consider removing it or
6525 * determine if it is necessary to add these flags to VMCPU_FF_HP_R0_PRE_HM_MASK. */
6526 /* Pending HM CR3 sync. */
6527 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6528 {
6529 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6530 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6531 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6532 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6533 }
6534
6535 /* Pending HM PAE PDPEs. */
6536 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6537 {
6538 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6539 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6540 }
6541#endif
6542
6543 /* Pending PGM C3 sync. */
6544 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6545 {
6546 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
6547 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6548 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6549 if (rcStrict2 != VINF_SUCCESS)
6550 {
6551 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6552 Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6553 return rcStrict2;
6554 }
6555 }
6556
6557 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6558 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6559 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6560 {
6561 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6562 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6563 Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6564 return rc2;
6565 }
6566
6567 /* Pending VM request packets, such as hardware interrupts. */
6568 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6569 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6570 {
6571 Log4Func(("Pending VM request forcing us back to ring-3\n"));
6572 return VINF_EM_PENDING_REQUEST;
6573 }
6574
6575 /* Pending PGM pool flushes. */
6576 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6577 {
6578 Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6579 return VINF_PGM_POOL_FLUSH_PENDING;
6580 }
6581
6582 /* Pending DMA requests. */
6583 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6584 {
6585 Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6586 return VINF_EM_RAW_TO_R3;
6587 }
6588
6589 return VINF_SUCCESS;
6590}
6591
6592
6593/**
6594 * Converts any TRPM trap into a pending HM event. This is typically used when
6595 * entering from ring-3 (not longjmp returns).
6596 *
6597 * @param pVCpu The cross context virtual CPU structure.
6598 */
6599static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6600{
6601 Assert(TRPMHasTrap(pVCpu));
6602 Assert(!pVCpu->hm.s.Event.fPending);
6603
6604 uint8_t uVector;
6605 TRPMEVENT enmTrpmEvent;
6606 RTGCUINT uErrCode;
6607 RTGCUINTPTR GCPtrFaultAddress;
6608 uint8_t cbInstr;
6609
6610 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6611 AssertRC(rc);
6612
6613 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6614 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6615 if (enmTrpmEvent == TRPM_TRAP)
6616 {
6617 switch (uVector)
6618 {
6619 case X86_XCPT_NMI:
6620 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6621 break;
6622
6623 case X86_XCPT_BP:
6624 case X86_XCPT_OF:
6625 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6626 break;
6627
6628 case X86_XCPT_PF:
6629 case X86_XCPT_DF:
6630 case X86_XCPT_TS:
6631 case X86_XCPT_NP:
6632 case X86_XCPT_SS:
6633 case X86_XCPT_GP:
6634 case X86_XCPT_AC:
6635 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6636 RT_FALL_THRU();
6637 default:
6638 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6639 break;
6640 }
6641 }
6642 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6643 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6644 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6645 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6646 else
6647 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6648
6649 rc = TRPMResetTrap(pVCpu);
6650 AssertRC(rc);
6651 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6652 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6653
6654 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6655}
6656
6657
6658/**
6659 * Converts the pending HM event into a TRPM trap.
6660 *
6661 * @param pVCpu The cross context virtual CPU structure.
6662 */
6663static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6664{
6665 Assert(pVCpu->hm.s.Event.fPending);
6666
6667 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6668 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6669 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6670 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6671
6672 /* If a trap was already pending, we did something wrong! */
6673 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6674
6675 TRPMEVENT enmTrapType;
6676 switch (uVectorType)
6677 {
6678 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6679 enmTrapType = TRPM_HARDWARE_INT;
6680 break;
6681
6682 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6683 enmTrapType = TRPM_SOFTWARE_INT;
6684 break;
6685
6686 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6687 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6688 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6689 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6690 enmTrapType = TRPM_TRAP;
6691 break;
6692
6693 default:
6694 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6695 enmTrapType = TRPM_32BIT_HACK;
6696 break;
6697 }
6698
6699 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6700
6701 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6702 AssertRC(rc);
6703
6704 if (fErrorCodeValid)
6705 TRPMSetErrorCode(pVCpu, uErrorCode);
6706
6707 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6708 && uVector == X86_XCPT_PF)
6709 {
6710 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6711 }
6712 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6713 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6714 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6715 {
6716 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6717 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6718 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6719 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6720 }
6721
6722 /* Clear any pending events from the VMCS. */
6723 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
6724 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
6725
6726 /* We're now done converting the pending event. */
6727 pVCpu->hm.s.Event.fPending = false;
6728}
6729
6730
6731/**
6732 * Does the necessary state syncing before returning to ring-3 for any reason
6733 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6734 *
6735 * @returns VBox status code.
6736 * @param pVCpu The cross context virtual CPU structure.
6737 * @param fImportState Whether to import the guest state from the VMCS back
6738 * to the guest-CPU context.
6739 *
6740 * @remarks No-long-jmp zone!!!
6741 */
6742static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
6743{
6744 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6745 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6746
6747 RTCPUID idCpu = RTMpCpuId();
6748 Log4Func(("HostCpuId=%u\n", idCpu));
6749
6750 /*
6751 * !!! IMPORTANT !!!
6752 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
6753 */
6754
6755 /* Save the guest state if necessary. */
6756 if (fImportState)
6757 {
6758 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
6759 AssertRCReturn(rc, rc);
6760 }
6761
6762 /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
6763 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
6764 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6765
6766 /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
6767#ifdef VBOX_STRICT
6768 if (CPUMIsHyperDebugStateActive(pVCpu))
6769 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6770#endif
6771 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6772 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6773 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6774
6775#if HC_ARCH_BITS == 64
6776 /* Restore host-state bits that VT-x only restores partially. */
6777 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6778 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6779 {
6780 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6781 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6782 }
6783 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6784#endif
6785
6786 /* Restore the lazy host MSRs as we're leaving VT-x context. */
6787 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
6788 {
6789 /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
6790 if (!fImportState)
6791 {
6792 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE
6793 | CPUMCTX_EXTRN_SYSCALL_MSRS);
6794 AssertRCReturn(rc, rc);
6795 }
6796 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6797 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
6798 }
6799 else
6800 pVCpu->hm.s.vmx.fLazyMsrs = 0;
6801
6802 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6803 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6804
6805 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6806 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
6807 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
6808 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6809 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6810 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6811 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6812 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6813 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6814
6815 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6816
6817 /** @todo This partially defeats the purpose of having preemption hooks.
6818 * The problem is, deregistering the hooks should be moved to a place that
6819 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6820 * context.
6821 */
6822 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6823 {
6824 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6825 AssertRCReturn(rc, rc);
6826
6827 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6828 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6829 }
6830 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6831 NOREF(idCpu);
6832
6833 return VINF_SUCCESS;
6834}
6835
6836
6837/**
6838 * Leaves the VT-x session.
6839 *
6840 * @returns VBox status code.
6841 * @param pVCpu The cross context virtual CPU structure.
6842 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6843 * out-of-sync. Make sure to update the required fields
6844 * before using them.
6845 *
6846 * @remarks No-long-jmp zone!!!
6847 */
6848static int hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6849{
6850 HM_DISABLE_PREEMPT();
6851 HMVMX_ASSERT_CPU_SAFE();
6852 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6853 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6854
6855 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6856 and done this from the VMXR0ThreadCtxCallback(). */
6857 if (!pVCpu->hm.s.fLeaveDone)
6858 {
6859 int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
6860 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
6861 pVCpu->hm.s.fLeaveDone = true;
6862 }
6863 Assert(!pMixedCtx->fExtrn); NOREF(pMixedCtx);
6864
6865 /*
6866 * !!! IMPORTANT !!!
6867 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6868 */
6869
6870 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6871 /** @todo Deregistering here means we need to VMCLEAR always
6872 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6873 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
6874 VMMR0ThreadCtxHookDisable(pVCpu);
6875
6876 /* Leave HM context. This takes care of local init (term). */
6877 int rc = HMR0LeaveCpu(pVCpu);
6878
6879 HM_RESTORE_PREEMPT();
6880 return rc;
6881}
6882
6883
6884/**
6885 * Does the necessary state syncing before doing a longjmp to ring-3.
6886 *
6887 * @returns VBox status code.
6888 * @param pVCpu The cross context virtual CPU structure.
6889 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6890 * out-of-sync. Make sure to update the required fields
6891 * before using them.
6892 *
6893 * @remarks No-long-jmp zone!!!
6894 */
6895DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6896{
6897 return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
6898}
6899
6900
6901/**
6902 * Take necessary actions before going back to ring-3.
6903 *
6904 * An action requires us to go back to ring-3. This function does the necessary
6905 * steps before we can safely return to ring-3. This is not the same as longjmps
6906 * to ring-3, this is voluntary and prepares the guest so it may continue
6907 * executing outside HM (recompiler/IEM).
6908 *
6909 * @returns VBox status code.
6910 * @param pVM The cross context VM structure.
6911 * @param pVCpu The cross context virtual CPU structure.
6912 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6913 * out-of-sync. Make sure to update the required fields
6914 * before using them.
6915 * @param rcExit The reason for exiting to ring-3. Can be
6916 * VINF_VMM_UNKNOWN_RING3_CALL.
6917 */
6918static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
6919{
6920 Assert(pVM);
6921 Assert(pVCpu);
6922 Assert(pMixedCtx);
6923 HMVMX_ASSERT_PREEMPT_SAFE();
6924
6925 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6926 {
6927 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6928 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6929 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6930 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6931 }
6932
6933 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6934 VMMRZCallRing3Disable(pVCpu);
6935 Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
6936
6937 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6938 if (pVCpu->hm.s.Event.fPending)
6939 {
6940 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6941 Assert(!pVCpu->hm.s.Event.fPending);
6942 }
6943
6944 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
6945 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
6946
6947 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
6948 and if we're injecting an event we should have a TRPM trap pending. */
6949 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6950#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
6951 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6952#endif
6953
6954 /* Save guest state and restore host state bits. */
6955 int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
6956 AssertRCReturn(rc, rc);
6957 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6958 /* Thread-context hooks are unregistered at this point!!! */
6959
6960 /* Sync recompiler state. */
6961 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6962 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6963 | CPUM_CHANGED_LDTR
6964 | CPUM_CHANGED_GDTR
6965 | CPUM_CHANGED_IDTR
6966 | CPUM_CHANGED_TR
6967 | CPUM_CHANGED_HIDDEN_SEL_REGS);
6968 if ( pVM->hm.s.fNestedPaging
6969 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
6970 {
6971 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6972 }
6973
6974 Assert(!pVCpu->hm.s.fClearTrapFlag);
6975
6976 /* Update the exit-to-ring 3 reason. */
6977 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
6978
6979 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6980 if (rcExit != VINF_EM_RAW_INTERRUPT)
6981 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
6982
6983 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6984
6985 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6986 VMMRZCallRing3RemoveNotification(pVCpu);
6987 VMMRZCallRing3Enable(pVCpu);
6988
6989 return rc;
6990}
6991
6992
6993/**
6994 * VMMRZCallRing3() callback wrapper which saves the guest state before we
6995 * longjump to ring-3 and possibly get preempted.
6996 *
6997 * @returns VBox status code.
6998 * @param pVCpu The cross context virtual CPU structure.
6999 * @param enmOperation The operation causing the ring-3 longjump.
7000 * @param pvUser Opaque pointer to the guest-CPU context. The data
7001 * may be out-of-sync. Make sure to update the required
7002 * fields before using them.
7003 */
7004static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7005{
7006 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7007 {
7008 /*
7009 * !!! IMPORTANT !!!
7010 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7011 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7012 */
7013 VMMRZCallRing3RemoveNotification(pVCpu);
7014 VMMRZCallRing3Disable(pVCpu);
7015 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7016 RTThreadPreemptDisable(&PreemptState);
7017
7018 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7019 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7020
7021#if HC_ARCH_BITS == 64
7022 /* Restore host-state bits that VT-x only restores partially. */
7023 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7024 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7025 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7026 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7027#endif
7028
7029 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7030 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7031 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7032
7033 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7034 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7035 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7036 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7037 {
7038 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7039 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7040 }
7041
7042 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7043 VMMR0ThreadCtxHookDisable(pVCpu);
7044 HMR0LeaveCpu(pVCpu);
7045 RTThreadPreemptRestore(&PreemptState);
7046 return VINF_SUCCESS;
7047 }
7048
7049 Assert(pVCpu);
7050 Assert(pvUser);
7051 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7052 HMVMX_ASSERT_PREEMPT_SAFE();
7053
7054 VMMRZCallRing3Disable(pVCpu);
7055 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7056
7057 Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7058
7059 int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7060 AssertRCReturn(rc, rc);
7061
7062 VMMRZCallRing3Enable(pVCpu);
7063 return VINF_SUCCESS;
7064}
7065
7066
7067/**
7068 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7069 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7070 *
7071 * @param pVCpu The cross context virtual CPU structure.
7072 */
7073DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7074{
7075 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7076 {
7077 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7078 {
7079 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7080 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7081 AssertRC(rc);
7082 Log4Func(("Setup interrupt-window exiting\n"));
7083 }
7084 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7085}
7086
7087
7088/**
7089 * Clears the interrupt-window exiting control in the VMCS.
7090 *
7091 * @param pVCpu The cross context virtual CPU structure.
7092 */
7093DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7094{
7095 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7096 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7097 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7098 AssertRC(rc);
7099 Log4Func(("Cleared interrupt-window exiting\n"));
7100}
7101
7102
7103/**
7104 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7105 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7106 *
7107 * @param pVCpu The cross context virtual CPU structure.
7108 */
7109DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7110{
7111 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7112 {
7113 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7114 {
7115 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7116 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7117 AssertRC(rc);
7118 Log4Func(("Setup NMI-window exiting\n"));
7119 }
7120 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7121}
7122
7123
7124/**
7125 * Clears the NMI-window exiting control in the VMCS.
7126 *
7127 * @param pVCpu The cross context virtual CPU structure.
7128 */
7129DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7130{
7131 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7132 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7133 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7134 AssertRC(rc);
7135 Log4Func(("Cleared NMI-window exiting\n"));
7136}
7137
7138
7139/**
7140 * Evaluates the event to be delivered to the guest and sets it as the pending
7141 * event.
7142 *
7143 * @returns The VT-x guest-interruptibility state.
7144 * @param pVCpu The cross context virtual CPU structure.
7145 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7146 * out-of-sync. Make sure to update the required fields
7147 * before using them.
7148 */
7149static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7150{
7151 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7152 uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7153 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7154 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7155 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7156
7157 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7158 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7159 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7160 Assert(!TRPMHasTrap(pVCpu));
7161
7162 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7163 APICUpdatePendingInterrupts(pVCpu);
7164
7165 /*
7166 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7167 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7168 */
7169 /** @todo SMI. SMIs take priority over NMIs. */
7170 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7171 {
7172 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7173 if ( !pVCpu->hm.s.Event.fPending
7174 && !fBlockNmi
7175 && !fBlockSti
7176 && !fBlockMovSS)
7177 {
7178 Log4Func(("Pending NMI\n"));
7179 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7180 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7181
7182 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7183 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7184 }
7185 else
7186 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7187 }
7188 /*
7189 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7190 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7191 */
7192 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7193 && !pVCpu->hm.s.fSingleInstruction)
7194 {
7195 Assert(!DBGFIsStepping(pVCpu));
7196 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7197 AssertRCReturn(rc, 0);
7198 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7199 if ( !pVCpu->hm.s.Event.fPending
7200 && !fBlockInt
7201 && !fBlockSti
7202 && !fBlockMovSS)
7203 {
7204 uint8_t u8Interrupt;
7205 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7206 if (RT_SUCCESS(rc))
7207 {
7208 Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7209 uint32_t u32IntInfo = u8Interrupt
7210 | VMX_EXIT_INTERRUPTION_INFO_VALID
7211 | (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7212
7213 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7214 }
7215 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7216 {
7217 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7218 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7219 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7220
7221 /*
7222 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7223 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7224 * need to re-set this force-flag here.
7225 */
7226 }
7227 else
7228 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7229 }
7230 else
7231 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7232 }
7233
7234 return fIntrState;
7235}
7236
7237
7238/**
7239 * Sets a pending-debug exception to be delivered to the guest if the guest is
7240 * single-stepping in the VMCS.
7241 *
7242 * @param pVCpu The cross context virtual CPU structure.
7243 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7244 * out-of-sync. Make sure to update the required fields
7245 * before using them.
7246 */
7247DECLINLINE(int) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7248{
7249 RT_NOREF(pVCpu);
7250 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS)); NOREF(pMixedCtx);
7251 return VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7252}
7253
7254
7255/**
7256 * Injects any pending events into the guest if the guest is in a state to
7257 * receive them.
7258 *
7259 * @returns Strict VBox status code (i.e. informational status codes too).
7260 * @param pVCpu The cross context virtual CPU structure.
7261 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7262 * out-of-sync. Make sure to update the required fields
7263 * before using them.
7264 * @param fIntrState The VT-x guest-interruptibility state.
7265 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7266 * return VINF_EM_DBG_STEPPED if the event was
7267 * dispatched directly.
7268 */
7269static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t fIntrState, bool fStepping)
7270{
7271 HMVMX_ASSERT_PREEMPT_SAFE();
7272 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7273
7274 bool fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7275 bool fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7276
7277 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7278 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7279 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7280 Assert(!TRPMHasTrap(pVCpu));
7281
7282 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7283 if (pVCpu->hm.s.Event.fPending)
7284 {
7285 /*
7286 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7287 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7288 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7289 *
7290 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7291 */
7292 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7293#ifdef VBOX_STRICT
7294 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7295 {
7296 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7297 Assert(!fBlockInt);
7298 Assert(!fBlockSti);
7299 Assert(!fBlockMovSS);
7300 }
7301 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7302 {
7303 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7304 Assert(!fBlockSti);
7305 Assert(!fBlockMovSS);
7306 Assert(!fBlockNmi);
7307 }
7308#endif
7309 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7310 (uint8_t)uIntType));
7311 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7312 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7313 &fIntrState);
7314 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7315
7316 /* Update the interruptibility-state as it could have been changed by
7317 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7318 fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7319 fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7320
7321 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7322 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7323 else
7324 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7325 }
7326
7327 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7328 if ( fBlockSti
7329 || fBlockMovSS)
7330 {
7331 if (!pVCpu->hm.s.fSingleInstruction)
7332 {
7333 /*
7334 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7335 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7336 * See Intel spec. 27.3.4 "Saving Non-Register State".
7337 */
7338 Assert(!DBGFIsStepping(pVCpu));
7339 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7340 AssertRCReturn(rc, rc);
7341 if (pMixedCtx->eflags.Bits.u1TF)
7342 {
7343 int rc2 = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
7344 AssertRCReturn(rc2, rc2);
7345 }
7346 }
7347 else if (pMixedCtx->eflags.Bits.u1TF)
7348 {
7349 /*
7350 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7351 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7352 */
7353 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7354 fIntrState = 0;
7355 }
7356 }
7357
7358 /*
7359 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7360 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7361 */
7362 int rc3 = hmR0VmxExportGuestIntrState(pVCpu, fIntrState);
7363 AssertRCReturn(rc3, rc3);
7364
7365 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7366 NOREF(fBlockMovSS); NOREF(fBlockSti);
7367 return rcStrict;
7368}
7369
7370
7371/**
7372 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7373 *
7374 * @param pVCpu The cross context virtual CPU structure.
7375 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7376 * out-of-sync. Make sure to update the required fields
7377 * before using them.
7378 */
7379DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7380{
7381 NOREF(pMixedCtx);
7382 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7383 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7384}
7385
7386
7387/**
7388 * Injects a double-fault (\#DF) exception into the VM.
7389 *
7390 * @returns Strict VBox status code (i.e. informational status codes too).
7391 * @param pVCpu The cross context virtual CPU structure.
7392 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7393 * out-of-sync. Make sure to update the required fields
7394 * before using them.
7395 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7396 * and should return VINF_EM_DBG_STEPPED if the event
7397 * is injected directly (register modified by us, not
7398 * by hardware on VM-entry).
7399 * @param pfIntrState Pointer to the current guest interruptibility-state.
7400 * This interruptibility-state will be updated if
7401 * necessary. This cannot not be NULL.
7402 */
7403DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCCPUMCTX pMixedCtx, bool fStepping, uint32_t *pfIntrState)
7404{
7405 NOREF(pMixedCtx);
7406 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7407 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7408 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7409 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */, fStepping,
7410 pfIntrState);
7411}
7412
7413
7414/**
7415 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7416 *
7417 * @param pVCpu The cross context virtual CPU structure.
7418 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7419 * out-of-sync. Make sure to update the required fields
7420 * before using them.
7421 */
7422DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7423{
7424 NOREF(pMixedCtx);
7425 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7426 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7427 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7428}
7429
7430
7431/**
7432 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7433 *
7434 * @param pVCpu The cross context virtual CPU structure.
7435 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7436 * out-of-sync. Make sure to update the required fields
7437 * before using them.
7438 * @param cbInstr The value of RIP that is to be pushed on the guest
7439 * stack.
7440 */
7441DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7442{
7443 NOREF(pMixedCtx);
7444 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7445 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7446 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7447}
7448
7449
7450/**
7451 * Injects a general-protection (\#GP) fault into the VM.
7452 *
7453 * @returns Strict VBox status code (i.e. informational status codes too).
7454 * @param pVCpu The cross context virtual CPU structure.
7455 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7456 * out-of-sync. Make sure to update the required fields
7457 * before using them.
7458 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7459 * mode, i.e. in real-mode it's not valid).
7460 * @param u32ErrorCode The error code associated with the \#GP.
7461 * @param fStepping Whether we're running in
7462 * hmR0VmxRunGuestCodeStep() and should return
7463 * VINF_EM_DBG_STEPPED if the event is injected
7464 * directly (register modified by us, not by
7465 * hardware on VM-entry).
7466 * @param pfIntrState Pointer to the current guest interruptibility-state.
7467 * This interruptibility-state will be updated if
7468 * necessary. This cannot not be NULL.
7469 */
7470DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7471 bool fStepping, uint32_t *pfIntrState)
7472{
7473 NOREF(pMixedCtx);
7474 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7475 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7476 if (fErrorCodeValid)
7477 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7478 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */, fStepping,
7479 pfIntrState);
7480}
7481
7482
7483#if 0 /* unused */
7484/**
7485 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7486 * VM.
7487 *
7488 * @param pVCpu The cross context virtual CPU structure.
7489 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7490 * out-of-sync. Make sure to update the required fields
7491 * before using them.
7492 * @param u32ErrorCode The error code associated with the \#GP.
7493 */
7494DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7495{
7496 NOREF(pMixedCtx);
7497 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7498 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7499 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7500 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7501}
7502#endif /* unused */
7503
7504
7505/**
7506 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7507 *
7508 * @param pVCpu The cross context virtual CPU structure.
7509 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7510 * out-of-sync. Make sure to update the required fields
7511 * before using them.
7512 * @param uVector The software interrupt vector number.
7513 * @param cbInstr The value of RIP that is to be pushed on the guest
7514 * stack.
7515 */
7516DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7517{
7518 NOREF(pMixedCtx);
7519 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7520 if ( uVector == X86_XCPT_BP
7521 || uVector == X86_XCPT_OF)
7522 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7523 else
7524 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7525 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7526}
7527
7528
7529/**
7530 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7531 * stack.
7532 *
7533 * @returns Strict VBox status code (i.e. informational status codes too).
7534 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7535 * @param pVM The cross context VM structure.
7536 * @param pMixedCtx Pointer to the guest-CPU context.
7537 * @param uValue The value to push to the guest stack.
7538 */
7539DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7540{
7541 /*
7542 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7543 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7544 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7545 */
7546 if (pMixedCtx->sp == 1)
7547 return VINF_EM_RESET;
7548 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7549 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7550 AssertRC(rc);
7551 return rc;
7552}
7553
7554
7555/**
7556 * Injects an event into the guest upon VM-entry by updating the relevant fields
7557 * in the VM-entry area in the VMCS.
7558 *
7559 * @returns Strict VBox status code (i.e. informational status codes too).
7560 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7561 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7562 *
7563 * @param pVCpu The cross context virtual CPU structure.
7564 * @param u64IntInfo The VM-entry interruption-information field.
7565 * @param cbInstr The VM-entry instruction length in bytes (for
7566 * software interrupts, exceptions and privileged
7567 * software exceptions).
7568 * @param u32ErrCode The VM-entry exception error code.
7569 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7570 * @param pfIntrState Pointer to the current guest interruptibility-state.
7571 * This interruptibility-state will be updated if
7572 * necessary. This cannot not be NULL.
7573 * @param fStepping Whether we're running in
7574 * hmR0VmxRunGuestCodeStep() and should return
7575 * VINF_EM_DBG_STEPPED if the event is injected
7576 * directly (register modified by us, not by
7577 * hardware on VM-entry).
7578 *
7579 * @remarks Requires CR0!
7580 */
7581static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7582 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7583{
7584 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7585 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7586 Assert(pfIntrState);
7587
7588 PCPUMCTX pMixedCtx = &pVCpu->cpum.GstCtx;
7589 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7590 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7591 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7592
7593#ifdef VBOX_STRICT
7594 /*
7595 * Validate the error-code-valid bit for hardware exceptions.
7596 * No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7597 */
7598 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7599 && !CPUMIsGuestInRealModeEx(pMixedCtx))
7600 {
7601 switch (uVector)
7602 {
7603 case X86_XCPT_PF:
7604 case X86_XCPT_DF:
7605 case X86_XCPT_TS:
7606 case X86_XCPT_NP:
7607 case X86_XCPT_SS:
7608 case X86_XCPT_GP:
7609 case X86_XCPT_AC:
7610 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7611 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7612 RT_FALL_THRU();
7613 default:
7614 break;
7615 }
7616 }
7617#endif
7618
7619 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7620 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7621 || !(*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7622
7623 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7624
7625 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7626 {
7627 /*
7628 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7629 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7630 */
7631 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7632 }
7633 else
7634 {
7635 /* We require CR0 to check if the guest is in real-mode. */
7636 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
7637 AssertRCReturn(rc, rc);
7638
7639 /*
7640 * Hardware interrupts & exceptions cannot be delivered through the software interrupt
7641 * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7642 * interrupt handler in the (real-mode) guest.
7643 *
7644 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7645 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7646 */
7647 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7648 {
7649 PVM pVM = pVCpu->CTX_SUFF(pVM);
7650 Assert(PDMVmmDevHeapIsEnabled(pVM));
7651 Assert(pVM->hm.s.vmx.pRealModeTSS);
7652
7653 /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7654 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK
7655 | CPUMCTX_EXTRN_TABLE_MASK
7656 | CPUMCTX_EXTRN_RIP
7657 | CPUMCTX_EXTRN_RSP
7658 | CPUMCTX_EXTRN_RFLAGS);
7659 AssertRCReturn(rc, rc);
7660
7661 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7662 size_t const cbIdtEntry = sizeof(X86IDTR16);
7663 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7664 {
7665 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7666 if (uVector == X86_XCPT_DF)
7667 return VINF_EM_RESET;
7668
7669 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7670 if (uVector == X86_XCPT_GP)
7671 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, pfIntrState);
7672
7673 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7674 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7675 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
7676 fStepping, pfIntrState);
7677 }
7678
7679 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7680 uint16_t uGuestIp = pMixedCtx->ip;
7681 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7682 {
7683 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7684 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7685 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7686 }
7687 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7688 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7689
7690 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7691 X86IDTR16 IdtEntry;
7692 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7693 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7694 AssertRCReturn(rc, rc);
7695
7696 /* Construct the stack frame for the interrupt/exception handler. */
7697 VBOXSTRICTRC rcStrict;
7698 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7699 if (rcStrict == VINF_SUCCESS)
7700 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7701 if (rcStrict == VINF_SUCCESS)
7702 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7703
7704 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7705 if (rcStrict == VINF_SUCCESS)
7706 {
7707 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7708 pMixedCtx->rip = IdtEntry.offSel;
7709 pMixedCtx->cs.Sel = IdtEntry.uSel;
7710 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7711 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7712 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7713 && uVector == X86_XCPT_PF)
7714 pMixedCtx->cr2 = GCPtrFaultAddress;
7715
7716 /* If any other guest-state bits are changed here, make sure to update
7717 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7718 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS
7719 | HM_CHANGED_GUEST_CR2
7720 | HM_CHANGED_GUEST_RIP
7721 | HM_CHANGED_GUEST_RFLAGS
7722 | HM_CHANGED_GUEST_RSP);
7723
7724 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7725 if (*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7726 {
7727 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7728 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7729 Log4Func(("Clearing inhibition due to STI\n"));
7730 *pfIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7731 }
7732 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7733 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7734
7735 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7736 it, if we are returning to ring-3 before executing guest code. */
7737 pVCpu->hm.s.Event.fPending = false;
7738
7739 /* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
7740 if (fStepping)
7741 rcStrict = VINF_EM_DBG_STEPPED;
7742 }
7743 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7744 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7745 return rcStrict;
7746 }
7747 }
7748
7749 /* Validate. */
7750 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7751 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7752 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7753
7754 /* Inject. */
7755 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7756 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7757 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7758 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7759 AssertRCReturn(rc, rc);
7760
7761 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7762 && uVector == X86_XCPT_PF)
7763 pMixedCtx->cr2 = GCPtrFaultAddress;
7764
7765 Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr,
7766 pMixedCtx->cr2));
7767
7768 return VINF_SUCCESS;
7769}
7770
7771
7772/**
7773 * Clears the interrupt-window exiting control in the VMCS and if necessary
7774 * clears the current event in the VMCS as well.
7775 *
7776 * @returns VBox status code.
7777 * @param pVCpu The cross context virtual CPU structure.
7778 *
7779 * @remarks Use this function only to clear events that have not yet been
7780 * delivered to the guest but are injected in the VMCS!
7781 * @remarks No-long-jump zone!!!
7782 */
7783static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
7784{
7785 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7786 {
7787 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7788 Log4Func(("Cleared interrupt widow\n"));
7789 }
7790
7791 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
7792 {
7793 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
7794 Log4Func(("Cleared interrupt widow\n"));
7795 }
7796}
7797
7798
7799/**
7800 * Enters the VT-x session.
7801 *
7802 * @returns VBox status code.
7803 * @param pVM The cross context VM structure.
7804 * @param pVCpu The cross context virtual CPU structure.
7805 * @param pCpu Pointer to the CPU info struct.
7806 */
7807VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7808{
7809 AssertPtr(pVM);
7810 AssertPtr(pVCpu);
7811 Assert(pVM->hm.s.vmx.fSupported);
7812 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7813 NOREF(pCpu); NOREF(pVM);
7814
7815 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7816 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7817 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7818
7819#ifdef VBOX_STRICT
7820 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
7821 RTCCUINTREG uHostCR4 = ASMGetCR4();
7822 if (!(uHostCR4 & X86_CR4_VMXE))
7823 {
7824 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7825 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7826 }
7827#endif
7828
7829 /*
7830 * Load the VCPU's VMCS as the current (and active) one.
7831 */
7832 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7833 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7834 if (RT_FAILURE(rc))
7835 return rc;
7836
7837 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7838 pVCpu->hm.s.fLeaveDone = false;
7839 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7840
7841 return VINF_SUCCESS;
7842}
7843
7844
7845/**
7846 * The thread-context callback (only on platforms which support it).
7847 *
7848 * @param enmEvent The thread-context event.
7849 * @param pVCpu The cross context virtual CPU structure.
7850 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7851 * @thread EMT(pVCpu)
7852 */
7853VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7854{
7855 NOREF(fGlobalInit);
7856
7857 switch (enmEvent)
7858 {
7859 case RTTHREADCTXEVENT_OUT:
7860 {
7861 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7862 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7863 VMCPU_ASSERT_EMT(pVCpu);
7864
7865 /* No longjmps (logger flushes, locks) in this fragile context. */
7866 VMMRZCallRing3Disable(pVCpu);
7867 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7868
7869 /*
7870 * Restore host-state (FPU, debug etc.)
7871 */
7872 if (!pVCpu->hm.s.fLeaveDone)
7873 {
7874 /*
7875 * Do -not- import the guest-state here as we might already be in the middle of importing
7876 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
7877 */
7878 hmR0VmxLeave(pVCpu, false /* fImportState */);
7879 pVCpu->hm.s.fLeaveDone = true;
7880 }
7881
7882 /* Leave HM context, takes care of local init (term). */
7883 int rc = HMR0LeaveCpu(pVCpu);
7884 AssertRC(rc); NOREF(rc);
7885
7886 /* Restore longjmp state. */
7887 VMMRZCallRing3Enable(pVCpu);
7888 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
7889 break;
7890 }
7891
7892 case RTTHREADCTXEVENT_IN:
7893 {
7894 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7895 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7896 VMCPU_ASSERT_EMT(pVCpu);
7897
7898 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7899 VMMRZCallRing3Disable(pVCpu);
7900 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7901
7902 /* Initialize the bare minimum state required for HM. This takes care of
7903 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7904 int rc = HMR0EnterCpu(pVCpu);
7905 AssertRC(rc);
7906 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7907 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7908
7909 /* Load the active VMCS as the current one. */
7910 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7911 {
7912 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7913 AssertRC(rc); NOREF(rc);
7914 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7915 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7916 }
7917 pVCpu->hm.s.fLeaveDone = false;
7918
7919 /* Restore longjmp state. */
7920 VMMRZCallRing3Enable(pVCpu);
7921 break;
7922 }
7923
7924 default:
7925 break;
7926 }
7927}
7928
7929
7930/**
7931 * Exports the host state into the VMCS host-state area.
7932 * Sets up the VM-exit MSR-load area.
7933 *
7934 * The CPU state will be loaded from these fields on every successful VM-exit.
7935 *
7936 * @returns VBox status code.
7937 * @param pVCpu The cross context virtual CPU structure.
7938 *
7939 * @remarks No-long-jump zone!!!
7940 */
7941static int hmR0VmxExportHostState(PVMCPU pVCpu)
7942{
7943 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7944
7945 int rc = VINF_SUCCESS;
7946 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
7947 {
7948 rc = hmR0VmxExportHostControlRegs();
7949 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7950
7951 rc = hmR0VmxExportHostSegmentRegs(pVCpu);
7952 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7953
7954 rc = hmR0VmxExportHostMsrs(pVCpu);
7955 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7956
7957 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
7958 }
7959 return rc;
7960}
7961
7962
7963/**
7964 * Saves the host state in the VMCS host-state.
7965 *
7966 * @returns VBox status code.
7967 * @param pVCpu The cross context virtual CPU structure.
7968 *
7969 * @remarks No-long-jump zone!!!
7970 */
7971VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
7972{
7973 AssertPtr(pVCpu);
7974 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7975
7976 /*
7977 * Export the host state here while entering HM context.
7978 * When thread-context hooks are used, we might get preempted and have to re-save the host
7979 * state but most of the time we won't be, so do it here before we disable interrupts.
7980 */
7981 return hmR0VmxExportHostState(pVCpu);
7982}
7983
7984
7985/**
7986 * Exports the guest state into the VMCS guest-state area.
7987 *
7988 * The will typically be done before VM-entry when the guest-CPU state and the
7989 * VMCS state may potentially be out of sync.
7990 *
7991 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
7992 * VM-entry controls.
7993 * Sets up the appropriate VMX non-root function to execute guest code based on
7994 * the guest CPU mode.
7995 *
7996 * @returns VBox strict status code.
7997 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
7998 * without unrestricted guest access and the VMMDev is not presently
7999 * mapped (e.g. EFI32).
8000 *
8001 * @param pVM The cross context VM structure.
8002 * @param pVCpu The cross context virtual CPU structure.
8003 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8004 * out-of-sync. Make sure to update the required fields
8005 * before using them.
8006 *
8007 * @remarks No-long-jump zone!!!
8008 */
8009static VBOXSTRICTRC hmR0VmxExportGuestState(PVM pVM, PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
8010{
8011 AssertPtr(pVM);
8012 AssertPtr(pVCpu);
8013 AssertPtr(pMixedCtx);
8014 HMVMX_ASSERT_PREEMPT_SAFE();
8015
8016 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8017
8018 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
8019
8020 /* Determine real-on-v86 mode. */
8021 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8022 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8023 && CPUMIsGuestInRealModeEx(pMixedCtx))
8024 {
8025 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8026 }
8027
8028 /*
8029 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8030 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8031 */
8032 int rc = hmR0VmxSelectVMRunHandler(pVCpu, pMixedCtx);
8033 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8034
8035 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8036 rc = hmR0VmxExportGuestEntryCtls(pVCpu, pMixedCtx);
8037 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8038
8039 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8040 rc = hmR0VmxExportGuestExitCtls(pVCpu, pMixedCtx);
8041 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8042
8043 rc = hmR0VmxExportGuestCR0(pVCpu, pMixedCtx);
8044 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8045
8046 VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pMixedCtx);
8047 if (rcStrict == VINF_SUCCESS)
8048 { /* likely */ }
8049 else
8050 {
8051 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8052 return rcStrict;
8053 }
8054
8055 rc = hmR0VmxExportGuestSegmentRegs(pVCpu, pMixedCtx);
8056 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8057
8058 /* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8059 may alter controls if we determine we don't have to swap EFER after all. */
8060 rc = hmR0VmxExportGuestMsrs(pVCpu, pMixedCtx);
8061 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8062
8063 rc = hmR0VmxExportGuestApicTpr(pVCpu);
8064 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8065
8066 /* This needs to be done after hmR0VmxExportGuestCR0() as it may alter intercepted exceptions. */
8067 rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8068 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8069
8070 /* Exporting RFLAGS here is fine, even though RFLAGS.TF might depend on guest debug state which is
8071 not exported here. It is re-evaluated and updated if necessary in hmR0VmxExportSharedState(). */
8072 rc = hmR0VmxExportGuestRip(pVCpu, pMixedCtx);
8073 rc |= hmR0VmxExportGuestRsp(pVCpu, pMixedCtx);
8074 rc |= hmR0VmxExportGuestRflags(pVCpu, pMixedCtx);
8075 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8076
8077 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8078 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8079 | HM_CHANGED_GUEST_CR2
8080 | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8081 | HM_CHANGED_GUEST_X87
8082 | HM_CHANGED_GUEST_SSE_AVX
8083 | HM_CHANGED_GUEST_OTHER_XSAVE
8084 | HM_CHANGED_GUEST_XCRx
8085 | HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8086 | HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8087 | HM_CHANGED_GUEST_TSC_AUX
8088 | HM_CHANGED_GUEST_OTHER_MSRS
8089 | HM_CHANGED_GUEST_HWVIRT
8090 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8091
8092 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8093 return rc;
8094}
8095
8096
8097/**
8098 * Exports the state shared between the host and guest into the VMCS.
8099 *
8100 * @param pVM The cross context VM structure.
8101 * @param pVCpu The cross context virtual CPU structure.
8102 * @param pCtx Pointer to the guest-CPU context.
8103 *
8104 * @remarks No-long-jump zone!!!
8105 */
8106static void hmR0VmxExportSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8107{
8108 NOREF(pVM);
8109
8110 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8111 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8112
8113 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8114 {
8115 int rc = hmR0VmxExportSharedDebugState(pVCpu, pCtx);
8116 AssertRC(rc);
8117 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8118
8119 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8120 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8121 {
8122 rc = hmR0VmxExportGuestRflags(pVCpu, pCtx);
8123 AssertRC(rc);
8124 }
8125 }
8126
8127 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8128 {
8129 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8130 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8131 }
8132
8133 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8134 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8135}
8136
8137
8138/**
8139 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8140 *
8141 * @returns Strict VBox status code (i.e. informational status codes too).
8142 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8143 * without unrestricted guest access and the VMMDev is not presently
8144 * mapped (e.g. EFI32).
8145 *
8146 * @param pVM The cross context VM structure.
8147 * @param pVCpu The cross context virtual CPU structure.
8148 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8149 * out-of-sync. Make sure to update the required fields
8150 * before using them.
8151 *
8152 * @remarks No-long-jump zone!!!
8153 */
8154static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
8155{
8156 HMVMX_ASSERT_PREEMPT_SAFE();
8157 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8158 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8159
8160#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8161 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_ALL_GUEST;
8162#endif
8163
8164 /*
8165 * For many exits it's only RIP that changes and hence try to export it first
8166 * without going through a lot of change flag checks.
8167 */
8168 VBOXSTRICTRC rcStrict;
8169 uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8170 if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8171 {
8172 rcStrict = hmR0VmxExportGuestRip(pVCpu, pMixedCtx);
8173 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8174 { /* likely */}
8175 else
8176 AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8177 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8178 }
8179 else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8180 {
8181 rcStrict = hmR0VmxExportGuestState(pVM, pVCpu, pMixedCtx);
8182 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8183 { /* likely */}
8184 else
8185 {
8186 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8187 VBOXSTRICTRC_VAL(rcStrict)));
8188 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8189 return rcStrict;
8190 }
8191 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8192 }
8193 else
8194 rcStrict = VINF_SUCCESS;
8195
8196#ifdef VBOX_STRICT
8197 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8198 fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8199 AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8200 ("fCtxChanged=%#RX64\n", fCtxChanged));
8201#endif
8202 return rcStrict;
8203}
8204
8205
8206/**
8207 * Does the preparations before executing guest code in VT-x.
8208 *
8209 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8210 * recompiler/IEM. We must be cautious what we do here regarding committing
8211 * guest-state information into the VMCS assuming we assuredly execute the
8212 * guest in VT-x mode.
8213 *
8214 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8215 * the common-state (TRPM/forceflags), we must undo those changes so that the
8216 * recompiler/IEM can (and should) use them when it resumes guest execution.
8217 * Otherwise such operations must be done when we can no longer exit to ring-3.
8218 *
8219 * @returns Strict VBox status code (i.e. informational status codes too).
8220 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8221 * have been disabled.
8222 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8223 * double-fault into the guest.
8224 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8225 * dispatched directly.
8226 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8227 *
8228 * @param pVM The cross context VM structure.
8229 * @param pVCpu The cross context virtual CPU structure.
8230 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8231 * out-of-sync. Make sure to update the required fields
8232 * before using them.
8233 * @param pVmxTransient Pointer to the VMX transient structure.
8234 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8235 * us ignore some of the reasons for returning to
8236 * ring-3, and return VINF_EM_DBG_STEPPED if event
8237 * dispatching took place.
8238 */
8239static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8240{
8241 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8242
8243#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8244 PGMRZDynMapFlushAutoSet(pVCpu);
8245#endif
8246
8247 /* Check force flag actions that might require us to go back to ring-3. */
8248 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8249 if (rcStrict == VINF_SUCCESS)
8250 { /* FFs doesn't get set all the time. */ }
8251 else
8252 return rcStrict;
8253
8254 /*
8255 * Setup the virtualized-APIC accesses.
8256 *
8257 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8258 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8259 *
8260 * This is the reason we do it here and not in hmR0VmxExportGuestState().
8261 */
8262 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8263 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8264 && PDMHasApic(pVM))
8265 {
8266 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8267 Assert(u64MsrApicBase);
8268 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8269
8270 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8271
8272 /* Unalias any existing mapping. */
8273 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8274 AssertRCReturn(rc, rc);
8275
8276 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8277 Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8278 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8279 AssertRCReturn(rc, rc);
8280
8281 /* Update the per-VCPU cache of the APIC base MSR. */
8282 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8283 }
8284
8285 if (TRPMHasTrap(pVCpu))
8286 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8287 uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8288
8289 /*
8290 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8291 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8292 */
8293 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, fIntrState, fStepping);
8294 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8295 { /* likely */ }
8296 else
8297 {
8298 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8299 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8300 return rcStrict;
8301 }
8302
8303 /*
8304 * No longjmps to ring-3 from this point on!!!
8305 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8306 * This also disables flushing of the R0-logger instance (if any).
8307 */
8308 VMMRZCallRing3Disable(pVCpu);
8309
8310 /*
8311 * Export the guest state bits.
8312 *
8313 * We cannot perform longjmps while loading the guest state because we do not preserve the
8314 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8315 * CPU migration.
8316 *
8317 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8318 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8319 * Hence, loading of the guest state needs to be done -after- injection of events.
8320 */
8321 rcStrict = hmR0VmxExportGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8322 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8323 { /* likely */ }
8324 else
8325 {
8326 VMMRZCallRing3Enable(pVCpu);
8327 return rcStrict;
8328 }
8329
8330 /*
8331 * We disable interrupts so that we don't miss any interrupts that would flag
8332 * preemption (IPI/timers etc.) when thread-context hooks aren't used and we've
8333 * been running with preemption disabled for a while. Since this is purly to aid
8334 * the RTThreadPreemptIsPending code, it doesn't matter that it may temporarily
8335 * reenable and disable interrupt on NT.
8336 *
8337 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8338 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8339 *
8340 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8341 * executing guest code.
8342 */
8343 pVmxTransient->fEFlags = ASMIntDisableFlags();
8344
8345 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8346 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8347 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8348 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8349 {
8350 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8351 {
8352 pVCpu->hm.s.Event.fPending = false;
8353
8354 /*
8355 * We've injected any pending events. This is really the point of no return (to ring-3).
8356 *
8357 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8358 * returns from this function, so don't enable them here.
8359 */
8360 return VINF_SUCCESS;
8361 }
8362
8363 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8364 rcStrict = VINF_EM_RAW_INTERRUPT;
8365 }
8366 else
8367 {
8368 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8369 rcStrict = VINF_EM_RAW_TO_R3;
8370 }
8371
8372 ASMSetFlags(pVmxTransient->fEFlags);
8373 VMMRZCallRing3Enable(pVCpu);
8374
8375 return rcStrict;
8376}
8377
8378
8379/**
8380 * Prepares to run guest code in VT-x and we've committed to doing so. This
8381 * means there is no backing out to ring-3 or anywhere else at this
8382 * point.
8383 *
8384 * @param pVM The cross context VM structure.
8385 * @param pVCpu The cross context virtual CPU structure.
8386 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8387 * out-of-sync. Make sure to update the required fields
8388 * before using them.
8389 * @param pVmxTransient Pointer to the VMX transient structure.
8390 *
8391 * @remarks Called with preemption disabled.
8392 * @remarks No-long-jump zone!!!
8393 */
8394static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8395{
8396 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8397 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8398 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8399
8400 /*
8401 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8402 */
8403 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8404 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8405
8406 if (!CPUMIsGuestFPUStateActive(pVCpu))
8407 {
8408 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8409 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8410 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
8411 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8412 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8413 }
8414
8415 /*
8416 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8417 */
8418 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8419 && pVCpu->hm.s.vmx.cMsrs > 0)
8420 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8421
8422 /*
8423 * Re-save the host state bits as we may've been preempted (only happens when
8424 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8425 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8426 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8427 * See @bugref{8432}.
8428 */
8429 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8430 {
8431 int rc = hmR0VmxExportHostState(pVCpu);
8432 AssertRC(rc);
8433 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8434 }
8435 Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8436
8437 /*
8438 * Export the state shared between host and guest (FPU, debug, lazy MSRs).
8439 */
8440 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8441 hmR0VmxExportSharedState(pVM, pVCpu, pMixedCtx);
8442 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8443
8444 /* Store status of the shared guest-host state at the time of VM-entry. */
8445#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8446 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8447 {
8448 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8449 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8450 }
8451 else
8452#endif
8453 {
8454 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8455 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8456 }
8457
8458 /*
8459 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8460 */
8461 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8462 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8463
8464 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8465 RTCPUID idCurrentCpu = pCpu->idCpu;
8466 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8467 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8468 {
8469 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8470 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8471 }
8472
8473 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8474 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8475 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8476 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8477
8478 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8479
8480 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8481 to start executing. */
8482
8483 /*
8484 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8485 */
8486 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8487 {
8488 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8489 {
8490 bool fMsrUpdated;
8491 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8492 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8493 &fMsrUpdated);
8494 AssertRC(rc2);
8495 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8496 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8497 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8498 }
8499 else
8500 {
8501 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8502 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8503 }
8504 }
8505
8506 if (pVM->cpum.ro.GuestFeatures.fIbrs)
8507 {
8508 bool fMsrUpdated;
8509 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8510 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8511 &fMsrUpdated);
8512 AssertRC(rc2);
8513 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8514 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8515 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8516 }
8517
8518#ifdef VBOX_STRICT
8519 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8520 hmR0VmxCheckHostEferMsr(pVCpu);
8521 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8522#endif
8523#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8524 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8525 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8526 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8527#endif
8528}
8529
8530
8531/**
8532 * Performs some essential restoration of state after running guest code in
8533 * VT-x.
8534 *
8535 * @param pVCpu The cross context virtual CPU structure.
8536 * @param pVmxTransient Pointer to the VMX transient structure.
8537 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8538 *
8539 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8540 *
8541 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8542 * unconditionally when it is safe to do so.
8543 */
8544static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8545{
8546 uint64_t const uHostTsc = ASMReadTSC();
8547 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8548
8549 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8550 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8551 pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8552 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8553 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8554 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8555
8556 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8557 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TSCOffset);
8558
8559 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8560 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8561 Assert(!ASMIntAreEnabled());
8562 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8563
8564#if HC_ARCH_BITS == 64
8565 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8566#endif
8567#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8568 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8569 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8570 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8571#else
8572 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8573#endif
8574#ifdef VBOX_STRICT
8575 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8576#endif
8577 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8578
8579 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8580 uint32_t uExitReason;
8581 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8582 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8583 AssertRC(rc);
8584 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8585 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8586
8587 if (rcVMRun == VINF_SUCCESS)
8588 {
8589 /*
8590 * Update the VM-exit history array here even if the VM-entry failed due to:
8591 * - Invalid guest state.
8592 * - MSR loading.
8593 * - Machine-check event.
8594 *
8595 * In any of the above cases we will still have a "valid" VM-exit reason
8596 * despite @a fVMEntryFailed being false.
8597 *
8598 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8599 *
8600 * Note! We don't have CS or RIP at this point. Will probably address that later
8601 * by amending the history entry added here.
8602 */
8603 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8604 UINT64_MAX, uHostTsc);
8605
8606 if (!pVmxTransient->fVMEntryFailed)
8607 {
8608 VMMRZCallRing3Enable(pVCpu);
8609
8610 /*
8611 * Import the guest-interruptibility state always as we need it while evaluating
8612 * injecting events on re-entry.
8613 *
8614 * We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8615 * checking for real-mode while exporting the state because all bits that cause
8616 * mode changes wrt CR0 are intercepted.
8617 */
8618 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8619 AssertRC(rc);
8620
8621#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8622 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8623 AssertRC(rc);
8624#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8625 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8626 AssertRC(rc);
8627#endif
8628
8629 /*
8630 * Sync the TPR shadow with our APIC state.
8631 */
8632 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8633 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8634 {
8635 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8636 AssertRC(rc);
8637 ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8638 }
8639
8640 return;
8641 }
8642 }
8643 else
8644 {
8645 Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8646 }
8647
8648 VMMRZCallRing3Enable(pVCpu);
8649}
8650
8651
8652/**
8653 * Runs the guest code using VT-x the normal way.
8654 *
8655 * @returns VBox status code.
8656 * @param pVM The cross context VM structure.
8657 * @param pVCpu The cross context virtual CPU structure.
8658 * @param pCtx Pointer to the guest-CPU context.
8659 *
8660 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8661 */
8662static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8663{
8664 VMXTRANSIENT VmxTransient;
8665 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8666 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8667 uint32_t cLoops = 0;
8668
8669 for (;; cLoops++)
8670 {
8671 Assert(!HMR0SuspendPending());
8672 HMVMX_ASSERT_CPU_SAFE();
8673
8674 /* Preparatory work for running guest code, this may force us to return
8675 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8676 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8677 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8678 if (rcStrict != VINF_SUCCESS)
8679 break;
8680
8681 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8682 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8683 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8684
8685 /* Restore any residual host-state and save any bits shared between host
8686 and guest into the guest-CPU state. Re-enables interrupts! */
8687 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8688
8689 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8690 if (RT_SUCCESS(rcRun))
8691 { /* very likely */ }
8692 else
8693 {
8694 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8695 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
8696 return rcRun;
8697 }
8698
8699 /* Profile the VM-exit. */
8700 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8701 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8702 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8703 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8704 HMVMX_START_EXIT_DISPATCH_PROF();
8705
8706 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8707
8708 /* Handle the VM-exit. */
8709#ifdef HMVMX_USE_FUNCTION_TABLE
8710 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8711#else
8712 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8713#endif
8714 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8715 if (rcStrict == VINF_SUCCESS)
8716 {
8717 if (cLoops <= pVM->hm.s.cMaxResumeLoops)
8718 continue; /* likely */
8719 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8720 rcStrict = VINF_EM_RAW_INTERRUPT;
8721 }
8722 break;
8723 }
8724
8725 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8726 return rcStrict;
8727}
8728
8729
8730
8731/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8732 * probes.
8733 *
8734 * The following few functions and associated structure contains the bloat
8735 * necessary for providing detailed debug events and dtrace probes as well as
8736 * reliable host side single stepping. This works on the principle of
8737 * "subclassing" the normal execution loop and workers. We replace the loop
8738 * method completely and override selected helpers to add necessary adjustments
8739 * to their core operation.
8740 *
8741 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8742 * any performance for debug and analysis features.
8743 *
8744 * @{
8745 */
8746
8747/**
8748 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
8749 * the debug run loop.
8750 */
8751typedef struct VMXRUNDBGSTATE
8752{
8753 /** The RIP we started executing at. This is for detecting that we stepped. */
8754 uint64_t uRipStart;
8755 /** The CS we started executing with. */
8756 uint16_t uCsStart;
8757
8758 /** Whether we've actually modified the 1st execution control field. */
8759 bool fModifiedProcCtls : 1;
8760 /** Whether we've actually modified the 2nd execution control field. */
8761 bool fModifiedProcCtls2 : 1;
8762 /** Whether we've actually modified the exception bitmap. */
8763 bool fModifiedXcptBitmap : 1;
8764
8765 /** We desire the modified the CR0 mask to be cleared. */
8766 bool fClearCr0Mask : 1;
8767 /** We desire the modified the CR4 mask to be cleared. */
8768 bool fClearCr4Mask : 1;
8769 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8770 uint32_t fCpe1Extra;
8771 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
8772 uint32_t fCpe1Unwanted;
8773 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
8774 uint32_t fCpe2Extra;
8775 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
8776 uint32_t bmXcptExtra;
8777 /** The sequence number of the Dtrace provider settings the state was
8778 * configured against. */
8779 uint32_t uDtraceSettingsSeqNo;
8780 /** VM-exits to check (one bit per VM-exit). */
8781 uint32_t bmExitsToCheck[3];
8782
8783 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
8784 uint32_t fProcCtlsInitial;
8785 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
8786 uint32_t fProcCtls2Initial;
8787 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
8788 uint32_t bmXcptInitial;
8789} VMXRUNDBGSTATE;
8790AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
8791typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
8792
8793
8794/**
8795 * Initializes the VMXRUNDBGSTATE structure.
8796 *
8797 * @param pVCpu The cross context virtual CPU structure of the
8798 * calling EMT.
8799 * @param pCtx The CPU register context to go with @a pVCpu.
8800 * @param pDbgState The structure to initialize.
8801 */
8802static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
8803{
8804 pDbgState->uRipStart = pCtx->rip;
8805 pDbgState->uCsStart = pCtx->cs.Sel;
8806
8807 pDbgState->fModifiedProcCtls = false;
8808 pDbgState->fModifiedProcCtls2 = false;
8809 pDbgState->fModifiedXcptBitmap = false;
8810 pDbgState->fClearCr0Mask = false;
8811 pDbgState->fClearCr4Mask = false;
8812 pDbgState->fCpe1Extra = 0;
8813 pDbgState->fCpe1Unwanted = 0;
8814 pDbgState->fCpe2Extra = 0;
8815 pDbgState->bmXcptExtra = 0;
8816 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
8817 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
8818 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
8819}
8820
8821
8822/**
8823 * Updates the VMSC fields with changes requested by @a pDbgState.
8824 *
8825 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
8826 * immediately before executing guest code, i.e. when interrupts are disabled.
8827 * We don't check status codes here as we cannot easily assert or return in the
8828 * latter case.
8829 *
8830 * @param pVCpu The cross context virtual CPU structure.
8831 * @param pDbgState The debug state.
8832 */
8833static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8834{
8835 /*
8836 * Ensure desired flags in VMCS control fields are set.
8837 * (Ignoring write failure here, as we're committed and it's just debug extras.)
8838 *
8839 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
8840 * there should be no stale data in pCtx at this point.
8841 */
8842 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
8843 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
8844 {
8845 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
8846 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
8847 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8848 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
8849 pDbgState->fModifiedProcCtls = true;
8850 }
8851
8852 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
8853 {
8854 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
8855 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
8856 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
8857 pDbgState->fModifiedProcCtls2 = true;
8858 }
8859
8860 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
8861 {
8862 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
8863 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8864 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
8865 pDbgState->fModifiedXcptBitmap = true;
8866 }
8867
8868 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
8869 {
8870 pVCpu->hm.s.vmx.u32CR0Mask = 0;
8871 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
8872 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
8873 }
8874
8875 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
8876 {
8877 pVCpu->hm.s.vmx.u32CR4Mask = 0;
8878 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
8879 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
8880 }
8881}
8882
8883
8884static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
8885{
8886 /*
8887 * Restore VM-exit control settings as we may not reenter this function the
8888 * next time around.
8889 */
8890 /* We reload the initial value, trigger what we can of recalculations the
8891 next time around. From the looks of things, that's all that's required atm. */
8892 if (pDbgState->fModifiedProcCtls)
8893 {
8894 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
8895 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
8896 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
8897 AssertRCReturn(rc2, rc2);
8898 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
8899 }
8900
8901 /* We're currently the only ones messing with this one, so just restore the
8902 cached value and reload the field. */
8903 if ( pDbgState->fModifiedProcCtls2
8904 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
8905 {
8906 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
8907 AssertRCReturn(rc2, rc2);
8908 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
8909 }
8910
8911 /* If we've modified the exception bitmap, we restore it and trigger
8912 reloading and partial recalculation the next time around. */
8913 if (pDbgState->fModifiedXcptBitmap)
8914 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
8915
8916 return rcStrict;
8917}
8918
8919
8920/**
8921 * Configures VM-exit controls for current DBGF and DTrace settings.
8922 *
8923 * This updates @a pDbgState and the VMCS execution control fields to reflect
8924 * the necessary VM-exits demanded by DBGF and DTrace.
8925 *
8926 * @param pVCpu The cross context virtual CPU structure.
8927 * @param pDbgState The debug state.
8928 * @param pVmxTransient Pointer to the VMX transient structure. May update
8929 * fUpdateTscOffsettingAndPreemptTimer.
8930 */
8931static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
8932{
8933 /*
8934 * Take down the dtrace serial number so we can spot changes.
8935 */
8936 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
8937 ASMCompilerBarrier();
8938
8939 /*
8940 * We'll rebuild most of the middle block of data members (holding the
8941 * current settings) as we go along here, so start by clearing it all.
8942 */
8943 pDbgState->bmXcptExtra = 0;
8944 pDbgState->fCpe1Extra = 0;
8945 pDbgState->fCpe1Unwanted = 0;
8946 pDbgState->fCpe2Extra = 0;
8947 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
8948 pDbgState->bmExitsToCheck[i] = 0;
8949
8950 /*
8951 * Software interrupts (INT XXh) - no idea how to trigger these...
8952 */
8953 PVM pVM = pVCpu->CTX_SUFF(pVM);
8954 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
8955 || VBOXVMM_INT_SOFTWARE_ENABLED())
8956 {
8957 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
8958 }
8959
8960 /*
8961 * INT3 breakpoints - triggered by #BP exceptions.
8962 */
8963 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
8964 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
8965
8966 /*
8967 * Exception bitmap and XCPT events+probes.
8968 */
8969 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
8970 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
8971 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
8972
8973 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
8974 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
8975 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
8976 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
8977 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
8978 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
8979 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
8980 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
8981 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
8982 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
8983 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
8984 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
8985 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
8986 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
8987 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
8988 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
8989 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
8990 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
8991
8992 if (pDbgState->bmXcptExtra)
8993 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
8994
8995 /*
8996 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
8997 *
8998 * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
8999 * So, when adding/changing/removing please don't forget to update it.
9000 *
9001 * Some of the macros are picking up local variables to save horizontal space,
9002 * (being able to see it in a table is the lesser evil here).
9003 */
9004#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9005 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9006 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9007#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9008 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9009 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9010 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9011 } else do { } while (0)
9012#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9013 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9014 { \
9015 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9016 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9017 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9018 } else do { } while (0)
9019#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9020 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9021 { \
9022 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9023 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9024 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9025 } else do { } while (0)
9026#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9027 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9028 { \
9029 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9030 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9031 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9032 } else do { } while (0)
9033
9034 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9035 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9036 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9037 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9038 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9039
9040 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9041 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9042 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9043 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9044 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9045 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9046 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9047 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9048 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9049 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9050 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9051 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9052 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9053 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9054 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9055 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9056 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9057 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9058 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9059 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9060 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9061 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9062 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9063 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9064 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9065 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9066 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9067 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9068 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9069 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9070 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9071 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9072 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9073 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9074 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9075 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9076
9077 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9078 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9079 {
9080 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
9081 | CPUMCTX_EXTRN_CR4
9082 | CPUMCTX_EXTRN_APIC_TPR);
9083 AssertRC(rc);
9084
9085#if 0 /** @todo fix me */
9086 pDbgState->fClearCr0Mask = true;
9087 pDbgState->fClearCr4Mask = true;
9088#endif
9089 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9090 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9091 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9092 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9093 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9094 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9095 require clearing here and in the loop if we start using it. */
9096 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9097 }
9098 else
9099 {
9100 if (pDbgState->fClearCr0Mask)
9101 {
9102 pDbgState->fClearCr0Mask = false;
9103 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9104 }
9105 if (pDbgState->fClearCr4Mask)
9106 {
9107 pDbgState->fClearCr4Mask = false;
9108 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9109 }
9110 }
9111 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9112 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9113
9114 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9115 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9116 {
9117 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9118 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9119 }
9120 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9121 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9122
9123 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9124 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9125 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9126 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9127 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9128 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9129 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9130 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9131#if 0 /** @todo too slow, fix handler. */
9132 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9133#endif
9134 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9135
9136 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9137 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9138 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9139 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9140 {
9141 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9142 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9143 }
9144 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9145 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9146 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9147 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9148
9149 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9150 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9151 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9152 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9153 {
9154 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9155 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9156 }
9157 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9158 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9159 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9160 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9161
9162 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9163 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9164 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9165 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9166 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9167 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9168 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9169 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9170 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9171 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9172 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9173 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9174 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9175 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9176 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9177 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9178 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9179 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9180 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9181 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9182 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9183 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9184
9185#undef IS_EITHER_ENABLED
9186#undef SET_ONLY_XBM_IF_EITHER_EN
9187#undef SET_CPE1_XBM_IF_EITHER_EN
9188#undef SET_CPEU_XBM_IF_EITHER_EN
9189#undef SET_CPE2_XBM_IF_EITHER_EN
9190
9191 /*
9192 * Sanitize the control stuff.
9193 */
9194 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9195 if (pDbgState->fCpe2Extra)
9196 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9197 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9198 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9199 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9200 {
9201 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9202 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9203 }
9204
9205 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9206 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9207 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9208 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9209}
9210
9211
9212/**
9213 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9214 * appropriate.
9215 *
9216 * The caller has checked the VM-exit against the
9217 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9218 * already, so we don't have to do that either.
9219 *
9220 * @returns Strict VBox status code (i.e. informational status codes too).
9221 * @param pVM The cross context VM structure.
9222 * @param pVCpu The cross context virtual CPU structure.
9223 * @param pMixedCtx Pointer to the guest-CPU context.
9224 * @param pVmxTransient Pointer to the VMX-transient structure.
9225 * @param uExitReason The VM-exit reason.
9226 *
9227 * @remarks The name of this function is displayed by dtrace, so keep it short
9228 * and to the point. No longer than 33 chars long, please.
9229 */
9230static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9231 PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9232{
9233 /*
9234 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9235 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9236 *
9237 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9238 * does. Must add/change/remove both places. Same ordering, please.
9239 *
9240 * Added/removed events must also be reflected in the next section
9241 * where we dispatch dtrace events.
9242 */
9243 bool fDtrace1 = false;
9244 bool fDtrace2 = false;
9245 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9246 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9247 uint32_t uEventArg = 0;
9248#define SET_EXIT(a_EventSubName) \
9249 do { \
9250 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9251 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9252 } while (0)
9253#define SET_BOTH(a_EventSubName) \
9254 do { \
9255 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9256 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9257 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9258 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9259 } while (0)
9260 switch (uExitReason)
9261 {
9262 case VMX_EXIT_MTF:
9263 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9264
9265 case VMX_EXIT_XCPT_OR_NMI:
9266 {
9267 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9268 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9269 {
9270 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9271 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9272 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9273 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9274 {
9275 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9276 {
9277 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9278 uEventArg = pVmxTransient->uExitIntErrorCode;
9279 }
9280 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9281 switch (enmEvent1)
9282 {
9283 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9284 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9285 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9286 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9287 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9288 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9289 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9290 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9291 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9292 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9293 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9294 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9295 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9296 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9297 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9298 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9299 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9300 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9301 default: break;
9302 }
9303 }
9304 else
9305 AssertFailed();
9306 break;
9307
9308 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9309 uEventArg = idxVector;
9310 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9311 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9312 break;
9313 }
9314 break;
9315 }
9316
9317 case VMX_EXIT_TRIPLE_FAULT:
9318 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9319 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9320 break;
9321 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9322 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9323 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9324 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9325 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9326
9327 /* Instruction specific VM-exits: */
9328 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9329 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9330 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9331 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9332 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9333 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9334 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9335 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9336 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9337 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9338 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9339 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9340 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9341 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9342 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9343 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9344 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9345 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9346 case VMX_EXIT_MOV_CRX:
9347 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9348 if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9349 SET_BOTH(CRX_READ);
9350 else
9351 SET_BOTH(CRX_WRITE);
9352 uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQualification);
9353 break;
9354 case VMX_EXIT_MOV_DRX:
9355 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9356 if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification)
9357 == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9358 SET_BOTH(DRX_READ);
9359 else
9360 SET_BOTH(DRX_WRITE);
9361 uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification);
9362 break;
9363 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9364 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9365 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9366 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9367 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9368 case VMX_EXIT_XDTR_ACCESS:
9369 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9370 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9371 {
9372 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9373 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9374 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9375 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9376 }
9377 break;
9378
9379 case VMX_EXIT_TR_ACCESS:
9380 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9381 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9382 {
9383 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9384 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9385 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9386 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9387 }
9388 break;
9389
9390 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9391 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9392 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9393 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9394 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9395 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9396 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9397 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9398 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9399 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9400 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9401
9402 /* Events that aren't relevant at this point. */
9403 case VMX_EXIT_EXT_INT:
9404 case VMX_EXIT_INT_WINDOW:
9405 case VMX_EXIT_NMI_WINDOW:
9406 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9407 case VMX_EXIT_PREEMPT_TIMER:
9408 case VMX_EXIT_IO_INSTR:
9409 break;
9410
9411 /* Errors and unexpected events. */
9412 case VMX_EXIT_INIT_SIGNAL:
9413 case VMX_EXIT_SIPI:
9414 case VMX_EXIT_IO_SMI:
9415 case VMX_EXIT_SMI:
9416 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9417 case VMX_EXIT_ERR_MSR_LOAD:
9418 case VMX_EXIT_ERR_MACHINE_CHECK:
9419 break;
9420
9421 default:
9422 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9423 break;
9424 }
9425#undef SET_BOTH
9426#undef SET_EXIT
9427
9428 /*
9429 * Dtrace tracepoints go first. We do them here at once so we don't
9430 * have to copy the guest state saving and stuff a few dozen times.
9431 * Down side is that we've got to repeat the switch, though this time
9432 * we use enmEvent since the probes are a subset of what DBGF does.
9433 */
9434 if (fDtrace1 || fDtrace2)
9435 {
9436 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9437 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9438 switch (enmEvent1)
9439 {
9440 /** @todo consider which extra parameters would be helpful for each probe. */
9441 case DBGFEVENT_END: break;
9442 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9443 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9444 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9445 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9446 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9447 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9448 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9449 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9450 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9451 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9452 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9453 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9454 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9455 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9456 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9457 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9458 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9459 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9460 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9461 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9462 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9463 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9464 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9465 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9466 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9467 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9468 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9469 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9470 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9471 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9472 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9473 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9474 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9475 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9476 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9477 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9478 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9479 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9480 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9481 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9482 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9483 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9484 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9485 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9486 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9487 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9488 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9489 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9490 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9491 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9492 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9493 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9494 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9495 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9496 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9497 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9498 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9499 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9500 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9501 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9502 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9503 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9504 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9505 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9506 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9507 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9508 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9509 }
9510 switch (enmEvent2)
9511 {
9512 /** @todo consider which extra parameters would be helpful for each probe. */
9513 case DBGFEVENT_END: break;
9514 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9515 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9516 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9517 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9518 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9519 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9520 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9521 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9522 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9523 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9524 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9525 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9526 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9527 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9528 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9529 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9530 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9531 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9532 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9533 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9534 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9535 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9536 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9537 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9538 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9539 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9540 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9541 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9542 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9543 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9544 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9545 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9546 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9547 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9548 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9549 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9550 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9551 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9552 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9553 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9554 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9555 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9556 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9557 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9558 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9559 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9560 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9561 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9562 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9563 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9564 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9565 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9566 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9567 }
9568 }
9569
9570 /*
9571 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9572 * the DBGF call will do a full check).
9573 *
9574 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9575 * Note! If we have to events, we prioritize the first, i.e. the instruction
9576 * one, in order to avoid event nesting.
9577 */
9578 if ( enmEvent1 != DBGFEVENT_END
9579 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9580 {
9581 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9582 if (rcStrict != VINF_SUCCESS)
9583 return rcStrict;
9584 }
9585 else if ( enmEvent2 != DBGFEVENT_END
9586 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9587 {
9588 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9589 if (rcStrict != VINF_SUCCESS)
9590 return rcStrict;
9591 }
9592
9593 return VINF_SUCCESS;
9594}
9595
9596
9597/**
9598 * Single-stepping VM-exit filtering.
9599 *
9600 * This is preprocessing the VM-exits and deciding whether we've gotten far
9601 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9602 * handling is performed.
9603 *
9604 * @returns Strict VBox status code (i.e. informational status codes too).
9605 * @param pVM The cross context VM structure.
9606 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9607 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9608 * out-of-sync. Make sure to update the required
9609 * fields before using them.
9610 * @param pVmxTransient Pointer to the VMX-transient structure.
9611 * @param uExitReason The VM-exit reason.
9612 * @param pDbgState The debug state.
9613 */
9614DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9615 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9616{
9617 /*
9618 * Expensive (saves context) generic dtrace VM-exit probe.
9619 */
9620 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9621 { /* more likely */ }
9622 else
9623 {
9624 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9625 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9626 AssertRC(rc);
9627 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9628 }
9629
9630 /*
9631 * Check for host NMI, just to get that out of the way.
9632 */
9633 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9634 { /* normally likely */ }
9635 else
9636 {
9637 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9638 AssertRCReturn(rc2, rc2);
9639 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9640 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9641 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9642 }
9643
9644 /*
9645 * Check for single stepping event if we're stepping.
9646 */
9647 if (pVCpu->hm.s.fSingleInstruction)
9648 {
9649 switch (uExitReason)
9650 {
9651 case VMX_EXIT_MTF:
9652 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9653
9654 /* Various events: */
9655 case VMX_EXIT_XCPT_OR_NMI:
9656 case VMX_EXIT_EXT_INT:
9657 case VMX_EXIT_TRIPLE_FAULT:
9658 case VMX_EXIT_INT_WINDOW:
9659 case VMX_EXIT_NMI_WINDOW:
9660 case VMX_EXIT_TASK_SWITCH:
9661 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9662 case VMX_EXIT_APIC_ACCESS:
9663 case VMX_EXIT_EPT_VIOLATION:
9664 case VMX_EXIT_EPT_MISCONFIG:
9665 case VMX_EXIT_PREEMPT_TIMER:
9666
9667 /* Instruction specific VM-exits: */
9668 case VMX_EXIT_CPUID:
9669 case VMX_EXIT_GETSEC:
9670 case VMX_EXIT_HLT:
9671 case VMX_EXIT_INVD:
9672 case VMX_EXIT_INVLPG:
9673 case VMX_EXIT_RDPMC:
9674 case VMX_EXIT_RDTSC:
9675 case VMX_EXIT_RSM:
9676 case VMX_EXIT_VMCALL:
9677 case VMX_EXIT_VMCLEAR:
9678 case VMX_EXIT_VMLAUNCH:
9679 case VMX_EXIT_VMPTRLD:
9680 case VMX_EXIT_VMPTRST:
9681 case VMX_EXIT_VMREAD:
9682 case VMX_EXIT_VMRESUME:
9683 case VMX_EXIT_VMWRITE:
9684 case VMX_EXIT_VMXOFF:
9685 case VMX_EXIT_VMXON:
9686 case VMX_EXIT_MOV_CRX:
9687 case VMX_EXIT_MOV_DRX:
9688 case VMX_EXIT_IO_INSTR:
9689 case VMX_EXIT_RDMSR:
9690 case VMX_EXIT_WRMSR:
9691 case VMX_EXIT_MWAIT:
9692 case VMX_EXIT_MONITOR:
9693 case VMX_EXIT_PAUSE:
9694 case VMX_EXIT_XDTR_ACCESS:
9695 case VMX_EXIT_TR_ACCESS:
9696 case VMX_EXIT_INVEPT:
9697 case VMX_EXIT_RDTSCP:
9698 case VMX_EXIT_INVVPID:
9699 case VMX_EXIT_WBINVD:
9700 case VMX_EXIT_XSETBV:
9701 case VMX_EXIT_RDRAND:
9702 case VMX_EXIT_INVPCID:
9703 case VMX_EXIT_VMFUNC:
9704 case VMX_EXIT_RDSEED:
9705 case VMX_EXIT_XSAVES:
9706 case VMX_EXIT_XRSTORS:
9707 {
9708 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
9709 | CPUMCTX_EXTRN_CS);
9710 AssertRCReturn(rc, rc);
9711 if ( pMixedCtx->rip != pDbgState->uRipStart
9712 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
9713 return VINF_EM_DBG_STEPPED;
9714 break;
9715 }
9716
9717 /* Errors and unexpected events: */
9718 case VMX_EXIT_INIT_SIGNAL:
9719 case VMX_EXIT_SIPI:
9720 case VMX_EXIT_IO_SMI:
9721 case VMX_EXIT_SMI:
9722 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9723 case VMX_EXIT_ERR_MSR_LOAD:
9724 case VMX_EXIT_ERR_MACHINE_CHECK:
9725 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9726 break;
9727
9728 default:
9729 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9730 break;
9731 }
9732 }
9733
9734 /*
9735 * Check for debugger event breakpoints and dtrace probes.
9736 */
9737 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9738 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9739 {
9740 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9741 if (rcStrict != VINF_SUCCESS)
9742 return rcStrict;
9743 }
9744
9745 /*
9746 * Normal processing.
9747 */
9748#ifdef HMVMX_USE_FUNCTION_TABLE
9749 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9750#else
9751 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9752#endif
9753}
9754
9755
9756/**
9757 * Single steps guest code using VT-x.
9758 *
9759 * @returns Strict VBox status code (i.e. informational status codes too).
9760 * @param pVM The cross context VM structure.
9761 * @param pVCpu The cross context virtual CPU structure.
9762 * @param pCtx Pointer to the guest-CPU context.
9763 *
9764 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
9765 */
9766static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9767{
9768 VMXTRANSIENT VmxTransient;
9769 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9770
9771 /* Set HMCPU indicators. */
9772 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
9773 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
9774 pVCpu->hm.s.fDebugWantRdTscExit = false;
9775 pVCpu->hm.s.fUsingDebugLoop = true;
9776
9777 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
9778 VMXRUNDBGSTATE DbgState;
9779 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
9780 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9781
9782 /*
9783 * The loop.
9784 */
9785 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9786 for (uint32_t cLoops = 0; ; cLoops++)
9787 {
9788 Assert(!HMR0SuspendPending());
9789 HMVMX_ASSERT_CPU_SAFE();
9790 bool fStepping = pVCpu->hm.s.fSingleInstruction;
9791
9792 /*
9793 * Preparatory work for running guest code, this may force us to return
9794 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
9795 */
9796 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9797 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
9798 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
9799 if (rcStrict != VINF_SUCCESS)
9800 break;
9801
9802 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9803 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
9804
9805 /*
9806 * Now we can run the guest code.
9807 */
9808 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9809
9810 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9811
9812 /*
9813 * Restore any residual host-state and save any bits shared between host
9814 * and guest into the guest-CPU state. Re-enables interrupts!
9815 */
9816 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
9817
9818 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9819 if (RT_SUCCESS(rcRun))
9820 { /* very likely */ }
9821 else
9822 {
9823 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9824 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9825 return rcRun;
9826 }
9827
9828 /* Profile the VM-exit. */
9829 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9830 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9831 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9832 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9833 HMVMX_START_EXIT_DISPATCH_PROF();
9834
9835 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9836
9837 /*
9838 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
9839 */
9840 rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
9841 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9842 if (rcStrict != VINF_SUCCESS)
9843 break;
9844 if (cLoops > pVM->hm.s.cMaxResumeLoops)
9845 {
9846 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9847 rcStrict = VINF_EM_RAW_INTERRUPT;
9848 break;
9849 }
9850
9851 /*
9852 * Stepping: Did the RIP change, if so, consider it a single step.
9853 * Otherwise, make sure one of the TFs gets set.
9854 */
9855 if (fStepping)
9856 {
9857 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
9858 | CPUMCTX_EXTRN_CS);
9859 AssertRC(rc);
9860 if ( pCtx->rip != DbgState.uRipStart
9861 || pCtx->cs.Sel != DbgState.uCsStart)
9862 {
9863 rcStrict = VINF_EM_DBG_STEPPED;
9864 break;
9865 }
9866 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
9867 }
9868
9869 /*
9870 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
9871 */
9872 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
9873 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9874 }
9875
9876 /*
9877 * Clear the X86_EFL_TF if necessary.
9878 */
9879 if (pVCpu->hm.s.fClearTrapFlag)
9880 {
9881 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
9882 AssertRC(rc);
9883 pVCpu->hm.s.fClearTrapFlag = false;
9884 pCtx->eflags.Bits.u1TF = 0;
9885 }
9886 /** @todo there seems to be issues with the resume flag when the monitor trap
9887 * flag is pending without being used. Seen early in bios init when
9888 * accessing APIC page in protected mode. */
9889
9890 /*
9891 * Restore VM-exit control settings as we may not reenter this function the
9892 * next time around.
9893 */
9894 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
9895
9896 /* Restore HMCPU indicators. */
9897 pVCpu->hm.s.fUsingDebugLoop = false;
9898 pVCpu->hm.s.fDebugWantRdTscExit = false;
9899 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
9900
9901 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9902 return rcStrict;
9903}
9904
9905
9906/** @} */
9907
9908
9909/**
9910 * Checks if any expensive dtrace probes are enabled and we should go to the
9911 * debug loop.
9912 *
9913 * @returns true if we should use debug loop, false if not.
9914 */
9915static bool hmR0VmxAnyExpensiveProbesEnabled(void)
9916{
9917 /* It's probably faster to OR the raw 32-bit counter variables together.
9918 Since the variables are in an array and the probes are next to one
9919 another (more or less), we have good locality. So, better read
9920 eight-nine cache lines ever time and only have one conditional, than
9921 128+ conditionals, right? */
9922 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
9923 | VBOXVMM_XCPT_DE_ENABLED_RAW()
9924 | VBOXVMM_XCPT_DB_ENABLED_RAW()
9925 | VBOXVMM_XCPT_BP_ENABLED_RAW()
9926 | VBOXVMM_XCPT_OF_ENABLED_RAW()
9927 | VBOXVMM_XCPT_BR_ENABLED_RAW()
9928 | VBOXVMM_XCPT_UD_ENABLED_RAW()
9929 | VBOXVMM_XCPT_NM_ENABLED_RAW()
9930 | VBOXVMM_XCPT_DF_ENABLED_RAW()
9931 | VBOXVMM_XCPT_TS_ENABLED_RAW()
9932 | VBOXVMM_XCPT_NP_ENABLED_RAW()
9933 | VBOXVMM_XCPT_SS_ENABLED_RAW()
9934 | VBOXVMM_XCPT_GP_ENABLED_RAW()
9935 | VBOXVMM_XCPT_PF_ENABLED_RAW()
9936 | VBOXVMM_XCPT_MF_ENABLED_RAW()
9937 | VBOXVMM_XCPT_AC_ENABLED_RAW()
9938 | VBOXVMM_XCPT_XF_ENABLED_RAW()
9939 | VBOXVMM_XCPT_VE_ENABLED_RAW()
9940 | VBOXVMM_XCPT_SX_ENABLED_RAW()
9941 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
9942 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
9943 ) != 0
9944 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
9945 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
9946 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
9947 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
9948 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
9949 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
9950 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
9951 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
9952 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
9953 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
9954 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
9955 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
9956 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
9957 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
9958 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
9959 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
9960 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
9961 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
9962 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
9963 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
9964 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
9965 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
9966 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
9967 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
9968 | VBOXVMM_INSTR_STR_ENABLED_RAW()
9969 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
9970 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
9971 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
9972 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
9973 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
9974 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
9975 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
9976 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
9977 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
9978 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
9979 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
9980 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
9981 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
9982 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
9983 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
9984 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
9985 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
9986 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
9987 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
9988 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
9989 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
9990 ) != 0
9991 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
9992 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
9993 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
9994 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
9995 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
9996 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
9997 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
9998 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
9999 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10000 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10001 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10002 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10003 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10004 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10005 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10006 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10007 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10008 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10009 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10010 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10011 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10012 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10013 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10014 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10015 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10016 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10017 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10018 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10019 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10020 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10021 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10022 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10023 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10024 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10025 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10026 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10027 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10028 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10029 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10030 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10031 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10032 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10033 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10034 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10035 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10036 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10037 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10038 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10039 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10040 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10041 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10042 ) != 0;
10043}
10044
10045
10046/**
10047 * Runs the guest code using VT-x.
10048 *
10049 * @returns Strict VBox status code (i.e. informational status codes too).
10050 * @param pVM The cross context VM structure.
10051 * @param pVCpu The cross context virtual CPU structure.
10052 * @param pCtx Pointer to the guest-CPU context.
10053 */
10054VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10055{
10056 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10057 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10058 HMVMX_ASSERT_PREEMPT_SAFE();
10059
10060 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10061
10062 VBOXSTRICTRC rcStrict;
10063 if ( !pVCpu->hm.s.fUseDebugLoop
10064 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10065 && !DBGFIsStepping(pVCpu)
10066 && !pVM->dbgf.ro.cEnabledInt3Breakpoints)
10067 rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10068 else
10069 rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10070
10071 if (rcStrict == VERR_EM_INTERPRETER)
10072 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10073 else if (rcStrict == VINF_EM_RESET)
10074 rcStrict = VINF_EM_TRIPLE_FAULT;
10075
10076 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10077 if (RT_FAILURE(rc2))
10078 {
10079 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10080 rcStrict = rc2;
10081 }
10082 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10083 return rcStrict;
10084}
10085
10086
10087#ifndef HMVMX_USE_FUNCTION_TABLE
10088DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10089{
10090#ifdef DEBUG_ramshankar
10091#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10092 do { \
10093 if (a_fSave != 0) \
10094 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10095 VBOXSTRICTRC rcStrict = a_CallExpr; \
10096 if (a_fSave != 0) \
10097 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10098 return rcStrict; \
10099 } while (0)
10100#else
10101# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10102#endif
10103 switch (rcReason)
10104 {
10105 case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10106 case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10107 case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10108 case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10109 case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10110 case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10111 case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10112 case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10113 case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10114 case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10115 case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10116 case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10117 case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10118 case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10119 case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10120 case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10121 case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10122 case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10123 case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10124 case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10125 case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10126 case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10127 case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10128 case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10129 case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10130 case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10131 case VMX_EXIT_XDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10132 case VMX_EXIT_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10133 case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10134 case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10135 case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10136 case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10137 case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10138 case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10139
10140 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10141 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10142 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10143 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10144 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10145 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10146 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10147 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10148 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10149
10150 case VMX_EXIT_VMCLEAR:
10151 case VMX_EXIT_VMLAUNCH:
10152 case VMX_EXIT_VMPTRLD:
10153 case VMX_EXIT_VMPTRST:
10154 case VMX_EXIT_VMREAD:
10155 case VMX_EXIT_VMRESUME:
10156 case VMX_EXIT_VMWRITE:
10157 case VMX_EXIT_VMXOFF:
10158 case VMX_EXIT_VMXON:
10159 case VMX_EXIT_INVEPT:
10160 case VMX_EXIT_INVVPID:
10161 case VMX_EXIT_VMFUNC:
10162 case VMX_EXIT_XSAVES:
10163 case VMX_EXIT_XRSTORS:
10164 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10165
10166 case VMX_EXIT_ENCLS:
10167 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10168 case VMX_EXIT_PML_FULL:
10169 default:
10170 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10171 }
10172#undef VMEXIT_CALL_RET
10173}
10174#endif /* !HMVMX_USE_FUNCTION_TABLE */
10175
10176
10177#ifdef VBOX_STRICT
10178/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10179# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10180 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10181
10182# define HMVMX_ASSERT_PREEMPT_CPUID() \
10183 do { \
10184 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10185 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10186 } while (0)
10187
10188# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10189 do { \
10190 AssertPtr(pVCpu); \
10191 AssertPtr(pMixedCtx); \
10192 AssertPtr(pVmxTransient); \
10193 Assert(pVmxTransient->fVMEntryFailed == false); \
10194 Assert(ASMIntAreEnabled()); \
10195 HMVMX_ASSERT_PREEMPT_SAFE(); \
10196 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10197 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10198 HMVMX_ASSERT_PREEMPT_SAFE(); \
10199 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10200 HMVMX_ASSERT_PREEMPT_CPUID(); \
10201 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10202 } while (0)
10203
10204# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10205 do { \
10206 Log4Func(("\n")); \
10207 } while (0)
10208#else /* nonstrict builds: */
10209# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10210 do { \
10211 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10212 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10213 } while (0)
10214# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10215#endif
10216
10217
10218/**
10219 * Advances the guest RIP by the specified number of bytes.
10220 *
10221 * @param pVCpu The cross context virtual CPU structure.
10222 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10223 * out-of-sync. Make sure to update the required fields
10224 * before using them.
10225 * @param cbInstr Number of bytes to advance the RIP by.
10226 *
10227 * @remarks No-long-jump zone!!!
10228 */
10229DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10230{
10231 /* Advance the RIP. */
10232 pMixedCtx->rip += cbInstr;
10233 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10234
10235 /* Update interrupt inhibition. */
10236 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10237 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10238 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10239}
10240
10241
10242/**
10243 * Advances the guest RIP after reading it from the VMCS.
10244 *
10245 * @returns VBox status code, no informational status codes.
10246 * @param pVCpu The cross context virtual CPU structure.
10247 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10248 * out-of-sync. Make sure to update the required fields
10249 * before using them.
10250 * @param pVmxTransient Pointer to the VMX transient structure.
10251 *
10252 * @remarks No-long-jump zone!!!
10253 */
10254static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10255{
10256 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10257 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
10258 | CPUMCTX_EXTRN_RFLAGS);
10259 AssertRCReturn(rc, rc);
10260
10261 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10262
10263 /*
10264 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10265 * pending debug exception field as it takes care of priority of events.
10266 *
10267 * See Intel spec. 32.2.1 "Debug Exceptions".
10268 */
10269 if ( !pVCpu->hm.s.fSingleInstruction
10270 && pMixedCtx->eflags.Bits.u1TF)
10271 {
10272 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
10273 AssertRCReturn(rc, rc);
10274 }
10275
10276 return VINF_SUCCESS;
10277}
10278
10279
10280/**
10281 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10282 * and update error record fields accordingly.
10283 *
10284 * @return VMX_IGS_* return codes.
10285 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10286 * wrong with the guest state.
10287 *
10288 * @param pVM The cross context VM structure.
10289 * @param pVCpu The cross context virtual CPU structure.
10290 * @param pCtx Pointer to the guest-CPU state.
10291 *
10292 * @remarks This function assumes our cache of the VMCS controls
10293 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10294 */
10295static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10296{
10297#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10298#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10299 uError = (err); \
10300 break; \
10301 } else do { } while (0)
10302
10303 int rc;
10304 uint32_t uError = VMX_IGS_ERROR;
10305 uint32_t u32Val;
10306 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10307
10308 do
10309 {
10310 /*
10311 * CR0.
10312 */
10313 uint32_t fSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10314 uint32_t fZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10315 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10316 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10317 if (fUnrestrictedGuest)
10318 fSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
10319
10320 uint32_t uGuestCR0;
10321 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uGuestCR0);
10322 AssertRCBreak(rc);
10323 HMVMX_CHECK_BREAK((uGuestCR0 & fSetCR0) == fSetCR0, VMX_IGS_CR0_FIXED1);
10324 HMVMX_CHECK_BREAK(!(uGuestCR0 & ~fZapCR0), VMX_IGS_CR0_FIXED0);
10325 if ( !fUnrestrictedGuest
10326 && (uGuestCR0 & X86_CR0_PG)
10327 && !(uGuestCR0 & X86_CR0_PE))
10328 {
10329 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10330 }
10331
10332 /*
10333 * CR4.
10334 */
10335 uint64_t fSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10336 uint64_t fZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10337
10338 uint32_t uGuestCR4;
10339 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uGuestCR4);
10340 AssertRCBreak(rc);
10341 HMVMX_CHECK_BREAK((uGuestCR4 & fSetCR4) == fSetCR4, VMX_IGS_CR4_FIXED1);
10342 HMVMX_CHECK_BREAK(!(uGuestCR4 & ~fZapCR4), VMX_IGS_CR4_FIXED0);
10343
10344 /*
10345 * IA32_DEBUGCTL MSR.
10346 */
10347 uint64_t u64Val;
10348 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10349 AssertRCBreak(rc);
10350 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10351 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10352 {
10353 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10354 }
10355 uint64_t u64DebugCtlMsr = u64Val;
10356
10357#ifdef VBOX_STRICT
10358 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10359 AssertRCBreak(rc);
10360 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10361#endif
10362 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10363
10364 /*
10365 * RIP and RFLAGS.
10366 */
10367 uint32_t u32Eflags;
10368#if HC_ARCH_BITS == 64
10369 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10370 AssertRCBreak(rc);
10371 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10372 if ( !fLongModeGuest
10373 || !pCtx->cs.Attr.n.u1Long)
10374 {
10375 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10376 }
10377 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10378 * must be identical if the "IA-32e mode guest" VM-entry
10379 * control is 1 and CS.L is 1. No check applies if the
10380 * CPU supports 64 linear-address bits. */
10381
10382 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10383 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10384 AssertRCBreak(rc);
10385 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10386 VMX_IGS_RFLAGS_RESERVED);
10387 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10388 u32Eflags = u64Val;
10389#else
10390 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10391 AssertRCBreak(rc);
10392 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10393 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10394#endif
10395
10396 if ( fLongModeGuest
10397 || ( fUnrestrictedGuest
10398 && !(uGuestCR0 & X86_CR0_PE)))
10399 {
10400 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10401 }
10402
10403 uint32_t u32EntryInfo;
10404 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10405 AssertRCBreak(rc);
10406 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10407 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10408 {
10409 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10410 }
10411
10412 /*
10413 * 64-bit checks.
10414 */
10415#if HC_ARCH_BITS == 64
10416 if (fLongModeGuest)
10417 {
10418 HMVMX_CHECK_BREAK(uGuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10419 HMVMX_CHECK_BREAK(uGuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10420 }
10421
10422 if ( !fLongModeGuest
10423 && (uGuestCR4 & X86_CR4_PCIDE))
10424 {
10425 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10426 }
10427
10428 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10429 * 51:32 beyond the processor's physical-address width are 0. */
10430
10431 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10432 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10433 {
10434 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10435 }
10436
10437 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10438 AssertRCBreak(rc);
10439 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10440
10441 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10442 AssertRCBreak(rc);
10443 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10444#endif
10445
10446 /*
10447 * PERF_GLOBAL MSR.
10448 */
10449 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10450 {
10451 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10452 AssertRCBreak(rc);
10453 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10454 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10455 }
10456
10457 /*
10458 * PAT MSR.
10459 */
10460 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10461 {
10462 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10463 AssertRCBreak(rc);
10464 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10465 for (unsigned i = 0; i < 8; i++)
10466 {
10467 uint8_t u8Val = (u64Val & 0xff);
10468 if ( u8Val != 0 /* UC */
10469 && u8Val != 1 /* WC */
10470 && u8Val != 4 /* WT */
10471 && u8Val != 5 /* WP */
10472 && u8Val != 6 /* WB */
10473 && u8Val != 7 /* UC- */)
10474 {
10475 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10476 }
10477 u64Val >>= 8;
10478 }
10479 }
10480
10481 /*
10482 * EFER MSR.
10483 */
10484 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10485 {
10486 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10487 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10488 AssertRCBreak(rc);
10489 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10490 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10491 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10492 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10493 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10494 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10495 || !(uGuestCR0 & X86_CR0_PG)
10496 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10497 VMX_IGS_EFER_LMA_LME_MISMATCH);
10498 }
10499
10500 /*
10501 * Segment registers.
10502 */
10503 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10504 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10505 if (!(u32Eflags & X86_EFL_VM))
10506 {
10507 /* CS */
10508 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10509 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10510 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10511 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10512 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10513 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10514 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10515 /* CS cannot be loaded with NULL in protected mode. */
10516 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10517 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10518 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10519 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10520 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10521 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10522 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10523 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10524 else
10525 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10526
10527 /* SS */
10528 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10529 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10530 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10531 if ( !(pCtx->cr0 & X86_CR0_PE)
10532 || pCtx->cs.Attr.n.u4Type == 3)
10533 {
10534 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10535 }
10536 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10537 {
10538 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10539 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10540 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10541 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10542 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10543 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10544 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10545 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10546 }
10547
10548 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
10549 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10550 {
10551 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10552 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10553 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10554 || pCtx->ds.Attr.n.u4Type > 11
10555 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10556 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10557 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10558 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10559 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10560 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10561 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10562 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10563 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10564 }
10565 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10566 {
10567 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10568 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10569 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10570 || pCtx->es.Attr.n.u4Type > 11
10571 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10572 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10573 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10574 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10575 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10576 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10577 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10578 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10579 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10580 }
10581 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10582 {
10583 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10584 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10585 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10586 || pCtx->fs.Attr.n.u4Type > 11
10587 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10588 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10589 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10590 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10591 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10592 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10593 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10594 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10595 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10596 }
10597 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10598 {
10599 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10600 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10601 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10602 || pCtx->gs.Attr.n.u4Type > 11
10603 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10604 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10605 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10606 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10607 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10608 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10609 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10610 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10611 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10612 }
10613 /* 64-bit capable CPUs. */
10614#if HC_ARCH_BITS == 64
10615 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10616 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10617 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10618 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10619 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10620 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10621 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10622 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10623 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10624 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10625 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10626#endif
10627 }
10628 else
10629 {
10630 /* V86 mode checks. */
10631 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10632 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10633 {
10634 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10635 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10636 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10637 }
10638 else
10639 {
10640 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10641 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10642 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10643 }
10644
10645 /* CS */
10646 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10647 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10648 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10649 /* SS */
10650 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10651 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10652 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10653 /* DS */
10654 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10655 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10656 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10657 /* ES */
10658 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10659 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10660 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10661 /* FS */
10662 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10663 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10664 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10665 /* GS */
10666 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10667 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10668 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10669 /* 64-bit capable CPUs. */
10670#if HC_ARCH_BITS == 64
10671 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10672 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10673 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10674 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10675 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10676 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10677 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10678 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10679 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10680 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10681 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10682#endif
10683 }
10684
10685 /*
10686 * TR.
10687 */
10688 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10689 /* 64-bit capable CPUs. */
10690#if HC_ARCH_BITS == 64
10691 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10692#endif
10693 if (fLongModeGuest)
10694 {
10695 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10696 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10697 }
10698 else
10699 {
10700 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10701 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10702 VMX_IGS_TR_ATTR_TYPE_INVALID);
10703 }
10704 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10705 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10706 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10707 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10708 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10709 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10710 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10711 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10712
10713 /*
10714 * GDTR and IDTR.
10715 */
10716#if HC_ARCH_BITS == 64
10717 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10718 AssertRCBreak(rc);
10719 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10720
10721 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10722 AssertRCBreak(rc);
10723 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10724#endif
10725
10726 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10727 AssertRCBreak(rc);
10728 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10729
10730 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10731 AssertRCBreak(rc);
10732 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10733
10734 /*
10735 * Guest Non-Register State.
10736 */
10737 /* Activity State. */
10738 uint32_t u32ActivityState;
10739 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10740 AssertRCBreak(rc);
10741 HMVMX_CHECK_BREAK( !u32ActivityState
10742 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
10743 VMX_IGS_ACTIVITY_STATE_INVALID);
10744 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10745 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10746 uint32_t u32IntrState;
10747 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
10748 AssertRCBreak(rc);
10749 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
10750 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10751 {
10752 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10753 }
10754
10755 /** @todo Activity state and injecting interrupts. Left as a todo since we
10756 * currently don't use activity states but ACTIVE. */
10757
10758 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10759 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10760
10761 /* Guest interruptibility-state. */
10762 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10763 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10764 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
10765 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10766 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10767 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10768 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10769 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10770 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10771 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
10772 {
10773 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10774 {
10775 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10776 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10777 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10778 }
10779 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10780 {
10781 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10782 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10783 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10784 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10785 }
10786 }
10787 /** @todo Assumes the processor is not in SMM. */
10788 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10789 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10790 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10791 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10792 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10793 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
10794 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10795 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10796 {
10797 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
10798 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10799 }
10800
10801 /* Pending debug exceptions. */
10802#if HC_ARCH_BITS == 64
10803 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
10804 AssertRCBreak(rc);
10805 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10806 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10807 u32Val = u64Val; /* For pending debug exceptions checks below. */
10808#else
10809 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
10810 AssertRCBreak(rc);
10811 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10812 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10813#endif
10814
10815 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10816 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
10817 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10818 {
10819 if ( (u32Eflags & X86_EFL_TF)
10820 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10821 {
10822 /* Bit 14 is PendingDebug.BS. */
10823 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10824 }
10825 if ( !(u32Eflags & X86_EFL_TF)
10826 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10827 {
10828 /* Bit 14 is PendingDebug.BS. */
10829 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10830 }
10831 }
10832
10833 /* VMCS link pointer. */
10834 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10835 AssertRCBreak(rc);
10836 if (u64Val != UINT64_C(0xffffffffffffffff))
10837 {
10838 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10839 /** @todo Bits beyond the processor's physical-address width MBZ. */
10840 /** @todo 32-bit located in memory referenced by value of this field (as a
10841 * physical address) must contain the processor's VMCS revision ID. */
10842 /** @todo SMM checks. */
10843 }
10844
10845 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10846 * not using Nested Paging? */
10847 if ( pVM->hm.s.fNestedPaging
10848 && !fLongModeGuest
10849 && CPUMIsGuestInPAEModeEx(pCtx))
10850 {
10851 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10852 AssertRCBreak(rc);
10853 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10854
10855 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10856 AssertRCBreak(rc);
10857 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10858
10859 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10860 AssertRCBreak(rc);
10861 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10862
10863 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10864 AssertRCBreak(rc);
10865 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10866 }
10867
10868 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10869 if (uError == VMX_IGS_ERROR)
10870 uError = VMX_IGS_REASON_NOT_FOUND;
10871 } while (0);
10872
10873 pVCpu->hm.s.u32HMError = uError;
10874 return uError;
10875
10876#undef HMVMX_ERROR_BREAK
10877#undef HMVMX_CHECK_BREAK
10878}
10879
10880/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10881/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
10882/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10883
10884/** @name VM-exit handlers.
10885 * @{
10886 */
10887
10888/**
10889 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
10890 */
10891HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10892{
10893 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10894 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
10895 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
10896 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
10897 return VINF_SUCCESS;
10898 return VINF_EM_RAW_INTERRUPT;
10899}
10900
10901
10902/**
10903 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
10904 */
10905HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10906{
10907 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10908 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
10909
10910 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10911 AssertRCReturn(rc, rc);
10912
10913 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10914 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
10915 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
10916 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
10917
10918 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10919 {
10920 /*
10921 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
10922 * injected it ourselves and anything we inject is not going to cause a VM-exit directly
10923 * for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
10924 *
10925 * [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
10926 * [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
10927 */
10928 VMXDispatchHostNmi();
10929 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
10930 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10931 return VINF_SUCCESS;
10932 }
10933
10934 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10935 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10936 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
10937 { /* likely */ }
10938 else
10939 {
10940 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
10941 rcStrictRc1 = VINF_SUCCESS;
10942 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10943 return rcStrictRc1;
10944 }
10945
10946 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
10947 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
10948 switch (uIntType)
10949 {
10950 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
10951 Assert(uVector == X86_XCPT_DB);
10952 RT_FALL_THRU();
10953 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
10954 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
10955 RT_FALL_THRU();
10956 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
10957 {
10958 /*
10959 * If there's any exception caused as a result of event injection, the resulting
10960 * secondary/final execption will be pending, we shall continue guest execution
10961 * after injecting the event. The page-fault case is complicated and we manually
10962 * handle any currently pending event in hmR0VmxExitXcptPF.
10963 */
10964 if (!pVCpu->hm.s.Event.fPending)
10965 { /* likely */ }
10966 else if (uVector != X86_XCPT_PF)
10967 {
10968 rc = VINF_SUCCESS;
10969 break;
10970 }
10971
10972 switch (uVector)
10973 {
10974 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
10975 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
10976 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
10977 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
10978 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
10979 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
10980
10981 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10982 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10983 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
10984 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10985 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
10986 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10987 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
10988 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10989 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
10990 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10991 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
10992 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10993 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
10994 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10995 default:
10996 {
10997 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
10998 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10999 {
11000 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11001 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11002 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11003
11004 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
11005 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11006 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11007 AssertRCReturn(rc, rc);
11008 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11009 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11010 0 /* GCPtrFaultAddress */);
11011 }
11012 else
11013 {
11014 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11015 pVCpu->hm.s.u32HMError = uVector;
11016 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11017 }
11018 break;
11019 }
11020 }
11021 break;
11022 }
11023
11024 default:
11025 {
11026 pVCpu->hm.s.u32HMError = uExitIntInfo;
11027 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11028 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11029 break;
11030 }
11031 }
11032 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11033 return rc;
11034}
11035
11036
11037/**
11038 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11039 */
11040HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11041{
11042 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11043
11044 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11045 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11046
11047 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11048 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11049 return VINF_SUCCESS;
11050}
11051
11052
11053/**
11054 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11055 */
11056HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11057{
11058 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11059 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11060 {
11061 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11062 HMVMX_RETURN_UNEXPECTED_EXIT();
11063 }
11064
11065 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11066
11067 /*
11068 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11069 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11070 */
11071 uint32_t fIntrState = 0;
11072 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11073 AssertRCReturn(rc, rc);
11074
11075 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11076 if ( fBlockSti
11077 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11078 {
11079 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11080 }
11081
11082 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11083 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11084
11085 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11086 return VINF_SUCCESS;
11087}
11088
11089
11090/**
11091 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11092 */
11093HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11094{
11095 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11096 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11097}
11098
11099
11100/**
11101 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11102 */
11103HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11104{
11105 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11106 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11107}
11108
11109
11110/**
11111 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11112 */
11113HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11114{
11115 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11116 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
11117
11118 /*
11119 * Get the state we need and update the exit history entry.
11120 */
11121 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11122 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
11123 | CPUMCTX_EXTRN_CS);
11124 AssertRCReturn(rc, rc);
11125
11126 VBOXSTRICTRC rcStrict;
11127 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11128 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
11129 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11130 if (!pExitRec)
11131 {
11132 /*
11133 * Regular CPUID instruction execution.
11134 */
11135 PVM pVM = pVCpu->CTX_SUFF(pVM);
11136 rcStrict = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11137 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11138 {
11139 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11140 Assert(pVmxTransient->cbInstr == 2);
11141 }
11142 else
11143 {
11144 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11145 rcStrict = VERR_EM_INTERPRETER;
11146 }
11147 }
11148 else
11149 {
11150 /*
11151 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
11152 */
11153 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
11154 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11155 AssertRCReturn(rc2, rc2);
11156
11157 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11158 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11159
11160 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11161 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11162
11163 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11164 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11165 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11166 }
11167 return VBOXSTRICTRC_TODO(rcStrict);
11168}
11169
11170
11171/**
11172 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11173 */
11174HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11175{
11176 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11177 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4);
11178 AssertRCReturn(rc, rc);
11179
11180 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11181 return VINF_EM_RAW_EMULATE_INSTR;
11182
11183 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11184 HMVMX_RETURN_UNEXPECTED_EXIT();
11185}
11186
11187
11188/**
11189 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11190 */
11191HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11192{
11193 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11194 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11195 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11196 AssertRCReturn(rc, rc);
11197
11198 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11199 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11200 {
11201 /* If we get a spurious VM-exit when offsetting is enabled,
11202 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11203 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11204 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11205 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
11206 | HM_CHANGED_GUEST_RFLAGS);
11207 }
11208 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11209 {
11210 rcStrict = VINF_SUCCESS;
11211 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11212 }
11213 return rcStrict;
11214}
11215
11216
11217/**
11218 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11219 */
11220HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11221{
11222 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11223 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
11224 | CPUMCTX_EXTRN_TSC_AUX);
11225 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11226 AssertRCReturn(rc, rc);
11227
11228 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11229 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11230 {
11231 /* If we get a spurious VM-exit when offsetting is enabled,
11232 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11233 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11234 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11235 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
11236 | HM_CHANGED_GUEST_RFLAGS);
11237 }
11238 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11239 {
11240 rcStrict = VINF_SUCCESS;
11241 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11242 }
11243 return rcStrict;
11244}
11245
11246
11247/**
11248 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11249 */
11250HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11251{
11252 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11253 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4
11254 | CPUMCTX_EXTRN_CR0
11255 | CPUMCTX_EXTRN_RFLAGS
11256 | CPUMCTX_EXTRN_SS);
11257 AssertRCReturn(rc, rc);
11258
11259 PVM pVM = pVCpu->CTX_SUFF(pVM);
11260 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11261 if (RT_LIKELY(rc == VINF_SUCCESS))
11262 {
11263 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11264 Assert(pVmxTransient->cbInstr == 2);
11265 }
11266 else
11267 {
11268 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11269 rc = VERR_EM_INTERPRETER;
11270 }
11271 return rc;
11272}
11273
11274
11275/**
11276 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11277 */
11278HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11279{
11280 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11281
11282 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11283 if (EMAreHypercallInstructionsEnabled(pVCpu))
11284 {
11285 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
11286 | CPUMCTX_EXTRN_RFLAGS
11287 | CPUMCTX_EXTRN_CR0
11288 | CPUMCTX_EXTRN_SS
11289 | CPUMCTX_EXTRN_CS
11290 | CPUMCTX_EXTRN_EFER);
11291 AssertRCReturn(rc, rc);
11292
11293 /* Perform the hypercall. */
11294 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11295 if (rcStrict == VINF_SUCCESS)
11296 {
11297 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11298 AssertRCReturn(rc, rc);
11299 }
11300 else
11301 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11302 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11303 || RT_FAILURE(rcStrict));
11304
11305 /* If the hypercall changes anything other than guest's general-purpose registers,
11306 we would need to reload the guest changed bits here before VM-entry. */
11307 }
11308 else
11309 Log4Func(("Hypercalls not enabled\n"));
11310
11311 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11312 if (RT_FAILURE(rcStrict))
11313 {
11314 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11315 rcStrict = VINF_SUCCESS;
11316 }
11317
11318 return rcStrict;
11319}
11320
11321
11322/**
11323 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11324 */
11325HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11326{
11327 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11328 PVM pVM = pVCpu->CTX_SUFF(pVM);
11329 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11330
11331 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11332 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK);
11333 AssertRCReturn(rc, rc);
11334
11335 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11336 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11337 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11338 else
11339 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11340 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11341 return rcStrict;
11342}
11343
11344
11345/**
11346 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11347 */
11348HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11349{
11350 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11351 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11352 | CPUMCTX_EXTRN_RFLAGS
11353 | CPUMCTX_EXTRN_SS);
11354 AssertRCReturn(rc, rc);
11355
11356 PVM pVM = pVCpu->CTX_SUFF(pVM);
11357 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11358 if (RT_LIKELY(rc == VINF_SUCCESS))
11359 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11360 else
11361 {
11362 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11363 rc = VERR_EM_INTERPRETER;
11364 }
11365 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11366 return rc;
11367}
11368
11369
11370/**
11371 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11372 */
11373HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11374{
11375 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11376 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11377 | CPUMCTX_EXTRN_RFLAGS
11378 | CPUMCTX_EXTRN_SS);
11379 AssertRCReturn(rc, rc);
11380
11381 PVM pVM = pVCpu->CTX_SUFF(pVM);
11382 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11383 rc = VBOXSTRICTRC_VAL(rc2);
11384 if (RT_LIKELY( rc == VINF_SUCCESS
11385 || rc == VINF_EM_HALT))
11386 {
11387 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11388 AssertRCReturn(rc3, rc3);
11389
11390 if ( rc == VINF_EM_HALT
11391 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11392 rc = VINF_SUCCESS;
11393 }
11394 else
11395 {
11396 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11397 rc = VERR_EM_INTERPRETER;
11398 }
11399 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11400 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11401 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11402 return rc;
11403}
11404
11405
11406/**
11407 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11408 */
11409HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11410{
11411 /*
11412 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11413 * mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11414 * treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11415 * VMX root operation. If we get here, something funny is going on.
11416 *
11417 * See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11418 */
11419 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11420 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11421 HMVMX_RETURN_UNEXPECTED_EXIT();
11422}
11423
11424
11425/**
11426 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11427 */
11428HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11429{
11430 /*
11431 * This can only happen if we support dual-monitor treatment of SMI, which can be activated
11432 * by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11433 * this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11434 * an SMI. If we get here, something funny is going on.
11435 *
11436 * See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11437 * See Intel spec. 25.3 "Other Causes of VM-Exits"
11438 */
11439 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11440 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11441 HMVMX_RETURN_UNEXPECTED_EXIT();
11442}
11443
11444
11445/**
11446 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11447 */
11448HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11449{
11450 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11451 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11452 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11453 HMVMX_RETURN_UNEXPECTED_EXIT();
11454}
11455
11456
11457/**
11458 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11459 */
11460HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11461{
11462 /*
11463 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11464 * We don't make use of it as our guests don't have direct access to the host LAPIC.
11465 * See Intel spec. 25.3 "Other Causes of VM-exits".
11466 */
11467 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11468 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11469 HMVMX_RETURN_UNEXPECTED_EXIT();
11470}
11471
11472
11473/**
11474 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11475 * VM-exit.
11476 */
11477HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11478{
11479 /*
11480 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11481 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11482 *
11483 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11484 * See Intel spec. "23.8 Restrictions on VMX operation".
11485 */
11486 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11487 return VINF_SUCCESS;
11488}
11489
11490
11491/**
11492 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11493 * VM-exit.
11494 */
11495HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11496{
11497 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11498 return VINF_EM_RESET;
11499}
11500
11501
11502/**
11503 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11504 */
11505HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11506{
11507 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11508 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11509
11510 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11511 AssertRCReturn(rc, rc);
11512
11513 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11514 rc = VINF_SUCCESS;
11515 else
11516 rc = VINF_EM_HALT;
11517
11518 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11519 if (rc != VINF_SUCCESS)
11520 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11521 return rc;
11522}
11523
11524
11525/**
11526 * VM-exit handler for instructions that result in a \#UD exception delivered to
11527 * the guest.
11528 */
11529HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11530{
11531 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11532 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11533 return VINF_SUCCESS;
11534}
11535
11536
11537/**
11538 * VM-exit handler for expiry of the VMX preemption timer.
11539 */
11540HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11541{
11542 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11543
11544 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11545 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11546
11547 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11548 PVM pVM = pVCpu->CTX_SUFF(pVM);
11549 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11550 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11551 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11552}
11553
11554
11555/**
11556 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11557 */
11558HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11559{
11560 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11561
11562 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11563 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
11564 | CPUMCTX_EXTRN_CR4);
11565 AssertRCReturn(rc, rc);
11566
11567 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11568 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11569 : HM_CHANGED_XCPT_RAISED_MASK);
11570
11571 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11572
11573 return rcStrict;
11574}
11575
11576
11577/**
11578 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11579 */
11580HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11581{
11582 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11583 /** @todo Use VM-exit instruction information. */
11584 return VERR_EM_INTERPRETER;
11585}
11586
11587
11588/**
11589 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11590 * Error VM-exit.
11591 */
11592HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11593{
11594 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11595 rc |= hmR0VmxCheckVmcsCtls(pVCpu);
11596 AssertRCReturn(rc, rc);
11597
11598 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11599 NOREF(uInvalidReason);
11600
11601#ifdef VBOX_STRICT
11602 uint32_t fIntrState;
11603 RTHCUINTREG uHCReg;
11604 uint64_t u64Val;
11605 uint32_t u32Val;
11606
11607 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11608 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11609 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11610 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11611 AssertRCReturn(rc, rc);
11612
11613 Log4(("uInvalidReason %u\n", uInvalidReason));
11614 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11615 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11616 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11617 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", fIntrState));
11618
11619 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11620 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11621 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11622 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11623 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11624 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11625 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11626 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11627 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11628 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11629 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11630 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11631#else
11632 NOREF(pVmxTransient);
11633#endif
11634
11635 hmR0DumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11636 return VERR_VMX_INVALID_GUEST_STATE;
11637}
11638
11639
11640/**
11641 * VM-exit handler for VM-entry failure due to an MSR-load
11642 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11643 */
11644HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11645{
11646 NOREF(pVmxTransient);
11647 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11648 HMVMX_RETURN_UNEXPECTED_EXIT();
11649}
11650
11651
11652/**
11653 * VM-exit handler for VM-entry failure due to a machine-check event
11654 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11655 */
11656HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11657{
11658 NOREF(pVmxTransient);
11659 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11660 HMVMX_RETURN_UNEXPECTED_EXIT();
11661}
11662
11663
11664/**
11665 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11666 * theory.
11667 */
11668HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11669{
11670 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
11671 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
11672 return VERR_VMX_UNDEFINED_EXIT_CODE;
11673}
11674
11675
11676/**
11677 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11678 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11679 * Conditional VM-exit.
11680 */
11681HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11682{
11683 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11684
11685 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11686 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11687 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11688 return VERR_EM_INTERPRETER;
11689 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11690 HMVMX_RETURN_UNEXPECTED_EXIT();
11691}
11692
11693
11694/**
11695 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11696 */
11697HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11698{
11699 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11700
11701 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11702 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11703 return VERR_EM_INTERPRETER;
11704 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11705 HMVMX_RETURN_UNEXPECTED_EXIT();
11706}
11707
11708
11709/**
11710 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11711 */
11712HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11713{
11714 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11715
11716 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
11717 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11718 | CPUMCTX_EXTRN_RFLAGS
11719 | CPUMCTX_EXTRN_SS);
11720 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11721 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
11722 AssertRCReturn(rc, rc);
11723 Log4Func(("ecx=%#RX32\n", pMixedCtx->ecx));
11724
11725#ifdef VBOX_STRICT
11726 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
11727 {
11728 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
11729 && pMixedCtx->ecx != MSR_K6_EFER)
11730 {
11731 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11732 pMixedCtx->ecx));
11733 HMVMX_RETURN_UNEXPECTED_EXIT();
11734 }
11735 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11736 {
11737 VMXMSREXITREAD enmRead;
11738 VMXMSREXITWRITE enmWrite;
11739 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
11740 AssertRCReturn(rc2, rc2);
11741 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11742 {
11743 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11744 HMVMX_RETURN_UNEXPECTED_EXIT();
11745 }
11746 }
11747 }
11748#endif
11749
11750 PVM pVM = pVCpu->CTX_SUFF(pVM);
11751 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11752 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
11753 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
11754 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11755 if (RT_SUCCESS(rc))
11756 {
11757 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11758 Assert(pVmxTransient->cbInstr == 2);
11759 }
11760 return rc;
11761}
11762
11763
11764/**
11765 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11766 */
11767HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11768{
11769 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11770 PVM pVM = pVCpu->CTX_SUFF(pVM);
11771 int rc = VINF_SUCCESS;
11772
11773 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
11774 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11775 | CPUMCTX_EXTRN_RFLAGS
11776 | CPUMCTX_EXTRN_SS);
11777 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11778 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
11779 AssertRCReturn(rc, rc);
11780 Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
11781
11782 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11783 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
11784 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
11785
11786 if (RT_SUCCESS(rc))
11787 {
11788 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11789
11790 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
11791 if ( pMixedCtx->ecx == MSR_IA32_APICBASE
11792 || ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
11793 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
11794 {
11795 /*
11796 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
11797 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
11798 * EMInterpretWrmsr() changes it.
11799 */
11800 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11801 }
11802 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
11803 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11804 else if (pMixedCtx->ecx == MSR_K6_EFER)
11805 {
11806 /*
11807 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
11808 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
11809 * the other bits as well, SCE and NXE. See @bugref{7368}.
11810 */
11811 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
11812 | HM_CHANGED_VMX_ENTRY_CTLS
11813 | HM_CHANGED_VMX_EXIT_CTLS);
11814 }
11815
11816 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
11817 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11818 {
11819 switch (pMixedCtx->ecx)
11820 {
11821 case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
11822 case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
11823 case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
11824 case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
11825 case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
11826 case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
11827 default:
11828 {
11829 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11830 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
11831 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11832 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
11833 break;
11834 }
11835 }
11836 }
11837#ifdef VBOX_STRICT
11838 else
11839 {
11840 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
11841 switch (pMixedCtx->ecx)
11842 {
11843 case MSR_IA32_SYSENTER_CS:
11844 case MSR_IA32_SYSENTER_EIP:
11845 case MSR_IA32_SYSENTER_ESP:
11846 case MSR_K8_FS_BASE:
11847 case MSR_K8_GS_BASE:
11848 {
11849 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
11850 HMVMX_RETURN_UNEXPECTED_EXIT();
11851 }
11852
11853 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
11854 default:
11855 {
11856 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11857 {
11858 /* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
11859 if (pMixedCtx->ecx != MSR_K6_EFER)
11860 {
11861 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11862 pMixedCtx->ecx));
11863 HMVMX_RETURN_UNEXPECTED_EXIT();
11864 }
11865 }
11866
11867 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11868 {
11869 VMXMSREXITREAD enmRead;
11870 VMXMSREXITWRITE enmWrite;
11871 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
11872 AssertRCReturn(rc2, rc2);
11873 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
11874 {
11875 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11876 HMVMX_RETURN_UNEXPECTED_EXIT();
11877 }
11878 }
11879 break;
11880 }
11881 }
11882 }
11883#endif /* VBOX_STRICT */
11884 }
11885 return rc;
11886}
11887
11888
11889/**
11890 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
11891 */
11892HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11893{
11894 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11895 /** @todo The guest has likely hit a contended spinlock. We might want to
11896 * poke a schedule different guest VCPU. */
11897 return VINF_EM_RAW_INTERRUPT;
11898}
11899
11900
11901/**
11902 * VM-exit handler for when the TPR value is lowered below the specified
11903 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
11904 */
11905HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11906{
11907 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11908 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
11909
11910 /*
11911 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
11912 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
11913 */
11914 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
11915 return VINF_SUCCESS;
11916}
11917
11918
11919/**
11920 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
11921 * VM-exit.
11922 *
11923 * @retval VINF_SUCCESS when guest execution can continue.
11924 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
11925 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
11926 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
11927 * interpreter.
11928 */
11929HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11930{
11931 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11932 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
11933
11934 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11935 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11936 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11937 AssertRCReturn(rc, rc);
11938
11939 VBOXSTRICTRC rcStrict;
11940 PVM pVM = pVCpu->CTX_SUFF(pVM);
11941 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
11942 uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQualification);
11943 switch (uAccessType)
11944 {
11945 case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
11946 {
11947 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
11948 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
11949 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification));
11950 AssertMsg( rcStrict == VINF_SUCCESS
11951 || rcStrict == VINF_IEM_RAISED_XCPT
11952 || rcStrict == VINF_PGM_CHANGE_MODE
11953 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11954
11955 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
11956 {
11957 case 0:
11958 {
11959 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11960 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
11961 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
11962 break;
11963 }
11964
11965 case 2:
11966 {
11967 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
11968 /* Nothing to do here, CR2 it's not part of the VMCS. */
11969 break;
11970 }
11971
11972 case 3:
11973 {
11974 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
11975 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
11976 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR3);
11977 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
11978 break;
11979 }
11980
11981 case 4:
11982 {
11983 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
11984 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11985 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
11986 pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
11987 break;
11988 }
11989
11990 case 8:
11991 {
11992 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
11993 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
11994 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11995 break;
11996 }
11997 default:
11998 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification)));
11999 break;
12000 }
12001 break;
12002 }
12003
12004 case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
12005 {
12006 Assert( !pVM->hm.s.fNestedPaging
12007 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12008 || pVCpu->hm.s.fUsingDebugLoop
12009 || VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 3);
12010 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12011 Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 8
12012 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12013
12014 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12015 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification),
12016 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification));
12017 AssertMsg( rcStrict == VINF_SUCCESS
12018 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12019#ifdef VBOX_WITH_STATISTICS
12020 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
12021 {
12022 case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
12023 case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
12024 case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
12025 case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
12026 case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
12027 }
12028#endif
12029 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
12030 VBOXSTRICTRC_VAL(rcStrict)));
12031 if (VMX_EXIT_QUAL_CRX_GENREG(uExitQualification) == X86_GREG_xSP)
12032 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RSP);
12033 break;
12034 }
12035
12036 case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12037 {
12038 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12039 AssertMsg( rcStrict == VINF_SUCCESS
12040 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12041
12042 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12043 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12044 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12045 break;
12046 }
12047
12048 case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12049 {
12050 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12051 VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQualification));
12052 AssertMsg( rcStrict == VINF_SUCCESS
12053 || rcStrict == VINF_IEM_RAISED_XCPT
12054 || rcStrict == VINF_PGM_CHANGE_MODE,
12055 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12056
12057 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12058 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12059 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12060 break;
12061 }
12062
12063 default:
12064 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12065 VERR_VMX_UNEXPECTED_EXCEPTION);
12066 }
12067
12068 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
12069 : HM_CHANGED_XCPT_RAISED_MASK);
12070 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12071 NOREF(pVM);
12072 return rcStrict;
12073}
12074
12075
12076/**
12077 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12078 * VM-exit.
12079 */
12080HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12081{
12082 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12083 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12084 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
12085
12086 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12087 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12088 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
12089 | CPUMCTX_EXTRN_SREG_MASK
12090 | CPUMCTX_EXTRN_EFER);
12091 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12092 AssertRCReturn(rc, rc);
12093
12094 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12095 uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQualification);
12096 uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQualification);
12097 bool fIOWrite = ( VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQualification)
12098 == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12099 bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQualification);
12100 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12101 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12102 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12103
12104 /*
12105 * Update exit history to see if this exit can be optimized.
12106 */
12107 VBOXSTRICTRC rcStrict;
12108 PCEMEXITREC pExitRec = NULL;
12109 if ( !fGstStepping
12110 && !fDbgStepping)
12111 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12112 !fIOString
12113 ? !fIOWrite
12114 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12115 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12116 : !fIOWrite
12117 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12118 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12119 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12120 if (!pExitRec)
12121 {
12122 /* I/O operation lookup arrays. */
12123 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12124 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12125 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12126 uint32_t const cbInstr = pVmxTransient->cbInstr;
12127 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12128 PVM pVM = pVCpu->CTX_SUFF(pVM);
12129 if (fIOString)
12130 {
12131 /*
12132 * INS/OUTS - I/O String instruction.
12133 *
12134 * Use instruction-information if available, otherwise fall back on
12135 * interpreting the instruction.
12136 */
12137 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12138 fIOWrite ? 'w' : 'r'));
12139 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12140 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12141 {
12142 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12143 AssertRCReturn(rc2, rc2);
12144 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12145 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12146 IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12147 bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification);
12148 if (fIOWrite)
12149 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12150 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12151 else
12152 {
12153 /*
12154 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12155 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12156 * See Intel Instruction spec. for "INS".
12157 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12158 */
12159 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12160 }
12161 }
12162 else
12163 rcStrict = IEMExecOne(pVCpu);
12164
12165 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12166 fUpdateRipAlready = true;
12167 }
12168 else
12169 {
12170 /*
12171 * IN/OUT - I/O instruction.
12172 */
12173 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12174 fIOWrite ? 'w' : 'r'));
12175 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12176 Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification));
12177 if (fIOWrite)
12178 {
12179 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12180 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12181 }
12182 else
12183 {
12184 uint32_t u32Result = 0;
12185 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12186 if (IOM_SUCCESS(rcStrict))
12187 {
12188 /* Save result of I/O IN instr. in AL/AX/EAX. */
12189 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12190 }
12191 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12192 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12193 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12194 }
12195 }
12196
12197 if (IOM_SUCCESS(rcStrict))
12198 {
12199 if (!fUpdateRipAlready)
12200 {
12201 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12202 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12203 }
12204
12205 /*
12206 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12207 * while booting Fedora 17 64-bit guest.
12208 *
12209 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12210 */
12211 if (fIOString)
12212 {
12213 /** @todo Single-step for INS/OUTS with REP prefix? */
12214 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12215 }
12216 else if ( !fDbgStepping
12217 && fGstStepping)
12218 {
12219 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
12220 AssertRCReturn(rc, rc);
12221 }
12222
12223 /*
12224 * If any I/O breakpoints are armed, we need to check if one triggered
12225 * and take appropriate action.
12226 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12227 */
12228 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12229 AssertRCReturn(rc, rc);
12230
12231 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12232 * execution engines about whether hyper BPs and such are pending. */
12233 uint32_t const uDr7 = pMixedCtx->dr[7];
12234 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12235 && X86_DR7_ANY_RW_IO(uDr7)
12236 && (pMixedCtx->cr4 & X86_CR4_DE))
12237 || DBGFBpIsHwIoArmed(pVM)))
12238 {
12239 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12240
12241 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12242 VMMRZCallRing3Disable(pVCpu);
12243 HM_DISABLE_PREEMPT();
12244
12245 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12246
12247 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12248 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12249 {
12250 /* Raise #DB. */
12251 if (fIsGuestDbgActive)
12252 ASMSetDR6(pMixedCtx->dr[6]);
12253 if (pMixedCtx->dr[7] != uDr7)
12254 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR7;
12255
12256 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12257 }
12258 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12259 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12260 else if ( rcStrict2 != VINF_SUCCESS
12261 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12262 rcStrict = rcStrict2;
12263 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12264
12265 HM_RESTORE_PREEMPT();
12266 VMMRZCallRing3Enable(pVCpu);
12267 }
12268 }
12269
12270#ifdef VBOX_STRICT
12271 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12272 Assert(!fIOWrite);
12273 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12274 Assert(fIOWrite);
12275 else
12276 {
12277# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12278 * statuses, that the VMM device and some others may return. See
12279 * IOM_SUCCESS() for guidance. */
12280 AssertMsg( RT_FAILURE(rcStrict)
12281 || rcStrict == VINF_SUCCESS
12282 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12283 || rcStrict == VINF_EM_DBG_BREAKPOINT
12284 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12285 || rcStrict == VINF_EM_RAW_TO_R3
12286 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12287# endif
12288 }
12289#endif
12290 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12291 }
12292 else
12293 {
12294 /*
12295 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12296 */
12297 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12298 AssertRCReturn(rc2, rc2);
12299 STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12300 : fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12301 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12302 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12303 VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification) ? "REP " : "",
12304 fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12305
12306 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12307 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12308
12309 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12310 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12311 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12312 }
12313 return rcStrict;
12314}
12315
12316
12317/**
12318 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12319 * VM-exit.
12320 */
12321HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12322{
12323 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12324
12325 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12326 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12327 AssertRCReturn(rc, rc);
12328 if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12329 {
12330 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12331 AssertRCReturn(rc, rc);
12332 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12333 {
12334 uint32_t uErrCode;
12335 RTGCUINTPTR GCPtrFaultAddress;
12336 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12337 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12338 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12339 if (fErrorCodeValid)
12340 {
12341 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12342 AssertRCReturn(rc, rc);
12343 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12344 }
12345 else
12346 uErrCode = 0;
12347
12348 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12349 && uVector == X86_XCPT_PF)
12350 GCPtrFaultAddress = pMixedCtx->cr2;
12351 else
12352 GCPtrFaultAddress = 0;
12353
12354 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12355 0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12356
12357 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12358 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12359 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12360 }
12361 }
12362
12363 /* Fall back to the interpreter to emulate the task-switch. */
12364 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12365 return VERR_EM_INTERPRETER;
12366}
12367
12368
12369/**
12370 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12371 */
12372HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12373{
12374 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12375 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12376 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12377 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12378 AssertRCReturn(rc, rc);
12379 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12380 return VINF_EM_DBG_STEPPED;
12381}
12382
12383
12384/**
12385 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12386 */
12387HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12388{
12389 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12390
12391 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12392
12393 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12394 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12395 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12396 {
12397 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12398 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12399 {
12400 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12401 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12402 }
12403 }
12404 else
12405 {
12406 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12407 rcStrict1 = VINF_SUCCESS;
12408 return rcStrict1;
12409 }
12410
12411 /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12412 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12413 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12414 AssertRCReturn(rc, rc);
12415
12416 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12417 uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12418 VBOXSTRICTRC rcStrict2;
12419 switch (uAccessType)
12420 {
12421 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12422 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12423 {
12424 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12425 || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
12426 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12427
12428 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12429 GCPhys &= PAGE_BASE_GC_MASK;
12430 GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12431 PVM pVM = pVCpu->CTX_SUFF(pVM);
12432 Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12433 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12434
12435 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12436 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12437 CPUMCTX2CORE(pMixedCtx), GCPhys);
12438 Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12439 if ( rcStrict2 == VINF_SUCCESS
12440 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12441 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12442 {
12443 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12444 | HM_CHANGED_GUEST_RSP
12445 | HM_CHANGED_GUEST_RFLAGS
12446 | HM_CHANGED_GUEST_APIC_TPR);
12447 rcStrict2 = VINF_SUCCESS;
12448 }
12449 break;
12450 }
12451
12452 default:
12453 Log4Func(("uAccessType=%#x\n", uAccessType));
12454 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12455 break;
12456 }
12457
12458 if (rcStrict2 != VINF_SUCCESS)
12459 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12460 return rcStrict2;
12461}
12462
12463
12464/**
12465 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12466 * VM-exit.
12467 */
12468HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12469{
12470 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12471
12472 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12473 if (pVmxTransient->fWasGuestDebugStateActive)
12474 {
12475 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12476 HMVMX_RETURN_UNEXPECTED_EXIT();
12477 }
12478
12479 if ( !pVCpu->hm.s.fSingleInstruction
12480 && !pVmxTransient->fWasHyperDebugStateActive)
12481 {
12482 Assert(!DBGFIsStepping(pVCpu));
12483 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12484
12485 /* Don't intercept MOV DRx any more. */
12486 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12487 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12488 AssertRCReturn(rc, rc);
12489
12490 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12491 VMMRZCallRing3Disable(pVCpu);
12492 HM_DISABLE_PREEMPT();
12493
12494 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12495 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12496 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12497
12498 HM_RESTORE_PREEMPT();
12499 VMMRZCallRing3Enable(pVCpu);
12500
12501#ifdef VBOX_WITH_STATISTICS
12502 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12503 AssertRCReturn(rc, rc);
12504 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12505 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12506 else
12507 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12508#endif
12509 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12510 return VINF_SUCCESS;
12511 }
12512
12513 /*
12514 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12515 * Update the segment registers and DR7 from the CPU.
12516 */
12517 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12518 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK
12519 | CPUMCTX_EXTRN_DR7);
12520 AssertRCReturn(rc, rc);
12521 Log4Func(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12522
12523 PVM pVM = pVCpu->CTX_SUFF(pVM);
12524 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12525 {
12526 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12527 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification),
12528 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification));
12529 if (RT_SUCCESS(rc))
12530 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12531 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12532 }
12533 else
12534 {
12535 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12536 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification),
12537 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification));
12538 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12539 }
12540
12541 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12542 if (RT_SUCCESS(rc))
12543 {
12544 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12545 AssertRCReturn(rc2, rc2);
12546 return VINF_SUCCESS;
12547 }
12548 return rc;
12549}
12550
12551
12552/**
12553 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12554 * Conditional VM-exit.
12555 */
12556HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12557{
12558 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12559 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12560
12561 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12562 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12563 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12564 {
12565 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12566 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12567 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12568 {
12569 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12570 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12571 }
12572 }
12573 else
12574 {
12575 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12576 rcStrict1 = VINF_SUCCESS;
12577 return rcStrict1;
12578 }
12579
12580 /*
12581 * Get sufficent state and update the exit history entry.
12582 */
12583 RTGCPHYS GCPhys;
12584 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12585 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12586 AssertRCReturn(rc, rc);
12587
12588 VBOXSTRICTRC rcStrict;
12589 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12590 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
12591 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12592 if (!pExitRec)
12593 {
12594 /*
12595 * If we succeed, resume guest execution.
12596 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12597 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12598 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12599 * weird case. See @bugref{6043}.
12600 */
12601 PVM pVM = pVCpu->CTX_SUFF(pVM);
12602 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12603 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12604 if ( rcStrict == VINF_SUCCESS
12605 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12606 || rcStrict == VERR_PAGE_NOT_PRESENT)
12607 {
12608 /* Successfully handled MMIO operation. */
12609 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12610 | HM_CHANGED_GUEST_RSP
12611 | HM_CHANGED_GUEST_RFLAGS
12612 | HM_CHANGED_GUEST_APIC_TPR);
12613 rcStrict = VINF_SUCCESS;
12614 }
12615 }
12616 else
12617 {
12618 /*
12619 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12620 */
12621 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
12622 int rc2 = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12623 AssertRCReturn(rc2, rc2);
12624
12625 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12626 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12627
12628 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12629 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12630
12631 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12632 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12633 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12634 }
12635 return VBOXSTRICTRC_TODO(rcStrict);
12636}
12637
12638
12639/**
12640 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12641 * VM-exit.
12642 */
12643HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12644{
12645 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12646 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12647
12648 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12649 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12650 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12651 {
12652 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12653 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12654 Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12655 }
12656 else
12657 {
12658 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12659 rcStrict1 = VINF_SUCCESS;
12660 return rcStrict1;
12661 }
12662
12663 RTGCPHYS GCPhys;
12664 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12665 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12666 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12667 AssertRCReturn(rc, rc);
12668
12669 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12670 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12671
12672 RTGCUINT uErrorCode = 0;
12673 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12674 uErrorCode |= X86_TRAP_PF_ID;
12675 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12676 uErrorCode |= X86_TRAP_PF_RW;
12677 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12678 uErrorCode |= X86_TRAP_PF_P;
12679
12680 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12681
12682 Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12683 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12684
12685 /* Handle the pagefault trap for the nested shadow table. */
12686 PVM pVM = pVCpu->CTX_SUFF(pVM);
12687 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12688 TRPMResetTrap(pVCpu);
12689
12690 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12691 if ( rcStrict2 == VINF_SUCCESS
12692 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12693 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12694 {
12695 /* Successfully synced our nested page tables. */
12696 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12697 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12698 | HM_CHANGED_GUEST_RSP
12699 | HM_CHANGED_GUEST_RFLAGS);
12700 return VINF_SUCCESS;
12701 }
12702
12703 Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12704 return rcStrict2;
12705}
12706
12707/** @} */
12708
12709/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12710/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
12711/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12712
12713/** @name VM-exit exception handlers.
12714 * @{
12715 */
12716
12717/**
12718 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12719 */
12720static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12721{
12722 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12723 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12724
12725 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
12726 AssertRCReturn(rc, rc);
12727
12728 if (!(pMixedCtx->cr0 & X86_CR0_NE))
12729 {
12730 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12731 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12732
12733 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12734 * provides VM-exit instruction length. If this causes problem later,
12735 * disassemble the instruction like it's done on AMD-V. */
12736 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12737 AssertRCReturn(rc2, rc2);
12738 return rc;
12739 }
12740
12741 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12742 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12743 return rc;
12744}
12745
12746
12747/**
12748 * VM-exit exception handler for \#BP (Breakpoint exception).
12749 */
12750static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12751{
12752 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12753 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12754
12755 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12756 AssertRCReturn(rc, rc);
12757
12758 rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx));
12759 if (rc == VINF_EM_RAW_GUEST_TRAP)
12760 {
12761 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12762 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12763 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12764 AssertRCReturn(rc, rc);
12765
12766 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12767 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12768 }
12769
12770 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
12771 return rc;
12772}
12773
12774
12775/**
12776 * VM-exit exception handler for \#AC (alignment check exception).
12777 */
12778static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12779{
12780 RT_NOREF_PV(pMixedCtx);
12781 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12782
12783 /*
12784 * Re-inject it. We'll detect any nesting before getting here.
12785 */
12786 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12787 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12788 AssertRCReturn(rc, rc);
12789 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
12790
12791 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12792 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12793 return VINF_SUCCESS;
12794}
12795
12796
12797/**
12798 * VM-exit exception handler for \#DB (Debug exception).
12799 */
12800static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12801{
12802 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12803 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
12804
12805 /*
12806 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
12807 * for processing.
12808 */
12809 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12810
12811 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
12812 uint64_t uDR6 = X86_DR6_INIT_VAL;
12813 uDR6 |= ( pVmxTransient->uExitQualification
12814 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
12815
12816 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
12817 Log6Func(("rc=%Rrc\n", rc));
12818 if (rc == VINF_EM_RAW_GUEST_TRAP)
12819 {
12820 /*
12821 * The exception was for the guest. Update DR6, DR7.GD and
12822 * IA32_DEBUGCTL.LBR before forwarding it.
12823 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
12824 */
12825 VMMRZCallRing3Disable(pVCpu);
12826 HM_DISABLE_PREEMPT();
12827
12828 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
12829 pMixedCtx->dr[6] |= uDR6;
12830 if (CPUMIsGuestDebugStateActive(pVCpu))
12831 ASMSetDR6(pMixedCtx->dr[6]);
12832
12833 HM_RESTORE_PREEMPT();
12834 VMMRZCallRing3Enable(pVCpu);
12835
12836 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12837 AssertRCReturn(rc, rc);
12838
12839 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
12840 pMixedCtx->dr[7] &= ~X86_DR7_GD;
12841
12842 /* Paranoia. */
12843 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
12844 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
12845
12846 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
12847 AssertRCReturn(rc, rc);
12848
12849 /*
12850 * Raise #DB in the guest.
12851 *
12852 * It is important to reflect exactly what the VM-exit gave us (preserving the
12853 * interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
12854 * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
12855 * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
12856 *
12857 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
12858 * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
12859 */
12860 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12861 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12862 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12863 AssertRCReturn(rc, rc);
12864 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12865 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12866 return VINF_SUCCESS;
12867 }
12868
12869 /*
12870 * Not a guest trap, must be a hypervisor related debug event then.
12871 * Update DR6 in case someone is interested in it.
12872 */
12873 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
12874 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
12875 CPUMSetHyperDR6(pVCpu, uDR6);
12876
12877 return rc;
12878}
12879
12880/**
12881 * VM-exit exception handler for \#GP (General-protection exception).
12882 *
12883 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
12884 */
12885static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12886{
12887 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12888 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
12889
12890 int rc;
12891 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
12892 { /* likely */ }
12893 else
12894 {
12895#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12896 Assert(pVCpu->hm.s.fUsingDebugLoop);
12897#endif
12898 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
12899 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12900 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12901 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12902 rc |= hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12903 AssertRCReturn(rc, rc);
12904 Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
12905 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
12906 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12907 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12908 return rc;
12909 }
12910
12911 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
12912 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
12913
12914 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
12915 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12916 AssertRCReturn(rc, rc);
12917
12918 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12919 uint32_t cbOp = 0;
12920 PVM pVM = pVCpu->CTX_SUFF(pVM);
12921 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12922 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
12923 if (RT_SUCCESS(rc))
12924 {
12925 rc = VINF_SUCCESS;
12926 Assert(cbOp == pDis->cbInstr);
12927 Log4Func(("Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12928 switch (pDis->pCurInstr->uOpcode)
12929 {
12930 case OP_CLI:
12931 {
12932 pMixedCtx->eflags.Bits.u1IF = 0;
12933 pMixedCtx->eflags.Bits.u1RF = 0;
12934 pMixedCtx->rip += pDis->cbInstr;
12935 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12936 if ( !fDbgStepping
12937 && pMixedCtx->eflags.Bits.u1TF)
12938 {
12939 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
12940 AssertRCReturn(rc, rc);
12941 }
12942 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
12943 break;
12944 }
12945
12946 case OP_STI:
12947 {
12948 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
12949 pMixedCtx->eflags.Bits.u1IF = 1;
12950 pMixedCtx->eflags.Bits.u1RF = 0;
12951 pMixedCtx->rip += pDis->cbInstr;
12952 if (!fOldIF)
12953 {
12954 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
12955 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
12956 }
12957 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12958 if ( !fDbgStepping
12959 && pMixedCtx->eflags.Bits.u1TF)
12960 {
12961 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
12962 AssertRCReturn(rc, rc);
12963 }
12964 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
12965 break;
12966 }
12967
12968 case OP_HLT:
12969 {
12970 rc = VINF_EM_HALT;
12971 pMixedCtx->rip += pDis->cbInstr;
12972 pMixedCtx->eflags.Bits.u1RF = 0;
12973 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12974 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
12975 break;
12976 }
12977
12978 case OP_POPF:
12979 {
12980 Log4Func(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12981 uint32_t cbParm;
12982 uint32_t uMask;
12983 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12984 if (pDis->fPrefix & DISPREFIX_OPSIZE)
12985 {
12986 cbParm = 4;
12987 uMask = 0xffffffff;
12988 }
12989 else
12990 {
12991 cbParm = 2;
12992 uMask = 0xffff;
12993 }
12994
12995 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
12996 RTGCPTR GCPtrStack = 0;
12997 X86EFLAGS Eflags;
12998 Eflags.u32 = 0;
12999 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13000 &GCPtrStack);
13001 if (RT_SUCCESS(rc))
13002 {
13003 Assert(sizeof(Eflags.u32) >= cbParm);
13004 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13005 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13006 }
13007 if (RT_FAILURE(rc))
13008 {
13009 rc = VERR_EM_INTERPRETER;
13010 break;
13011 }
13012 Log4Func(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13013 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13014 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13015 pMixedCtx->esp += cbParm;
13016 pMixedCtx->esp &= uMask;
13017 pMixedCtx->rip += pDis->cbInstr;
13018 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13019 | HM_CHANGED_GUEST_RSP
13020 | HM_CHANGED_GUEST_RFLAGS);
13021 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13022 POPF restores EFLAGS.TF. */
13023 if ( !fDbgStepping
13024 && fGstStepping)
13025 {
13026 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13027 AssertRCReturn(rc, rc);
13028 }
13029 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13030 break;
13031 }
13032
13033 case OP_PUSHF:
13034 {
13035 uint32_t cbParm;
13036 uint32_t uMask;
13037 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13038 {
13039 cbParm = 4;
13040 uMask = 0xffffffff;
13041 }
13042 else
13043 {
13044 cbParm = 2;
13045 uMask = 0xffff;
13046 }
13047
13048 /* Get the stack pointer & push the contents of eflags onto the stack. */
13049 RTGCPTR GCPtrStack = 0;
13050 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13051 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13052 if (RT_FAILURE(rc))
13053 {
13054 rc = VERR_EM_INTERPRETER;
13055 break;
13056 }
13057 X86EFLAGS Eflags = pMixedCtx->eflags;
13058 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13059 Eflags.Bits.u1RF = 0;
13060 Eflags.Bits.u1VM = 0;
13061
13062 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13063 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13064 {
13065 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13066 rc = VERR_EM_INTERPRETER;
13067 break;
13068 }
13069 Log4Func(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13070 pMixedCtx->esp -= cbParm;
13071 pMixedCtx->esp &= uMask;
13072 pMixedCtx->rip += pDis->cbInstr;
13073 pMixedCtx->eflags.Bits.u1RF = 0;
13074 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13075 | HM_CHANGED_GUEST_RSP
13076 | HM_CHANGED_GUEST_RFLAGS);
13077 if ( !fDbgStepping
13078 && pMixedCtx->eflags.Bits.u1TF)
13079 {
13080 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13081 AssertRCReturn(rc, rc);
13082 }
13083 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13084 break;
13085 }
13086
13087 case OP_IRET:
13088 {
13089 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13090 * instruction reference. */
13091 RTGCPTR GCPtrStack = 0;
13092 uint32_t uMask = 0xffff;
13093 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13094 uint16_t aIretFrame[3];
13095 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13096 {
13097 rc = VERR_EM_INTERPRETER;
13098 break;
13099 }
13100 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13101 &GCPtrStack);
13102 if (RT_SUCCESS(rc))
13103 {
13104 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13105 PGMACCESSORIGIN_HM));
13106 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13107 }
13108 if (RT_FAILURE(rc))
13109 {
13110 rc = VERR_EM_INTERPRETER;
13111 break;
13112 }
13113 pMixedCtx->eip = 0;
13114 pMixedCtx->ip = aIretFrame[0];
13115 pMixedCtx->cs.Sel = aIretFrame[1];
13116 pMixedCtx->cs.ValidSel = aIretFrame[1];
13117 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13118 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13119 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13120 pMixedCtx->sp += sizeof(aIretFrame);
13121 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13122 | HM_CHANGED_GUEST_CS
13123 | HM_CHANGED_GUEST_RSP
13124 | HM_CHANGED_GUEST_RFLAGS);
13125 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13126 if ( !fDbgStepping
13127 && fGstStepping)
13128 {
13129 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13130 AssertRCReturn(rc, rc);
13131 }
13132 Log4Func(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13133 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13134 break;
13135 }
13136
13137 case OP_INT:
13138 {
13139 uint16_t uVector = pDis->Param1.uValue & 0xff;
13140 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13141 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13142 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13143 break;
13144 }
13145
13146 case OP_INTO:
13147 {
13148 if (pMixedCtx->eflags.Bits.u1OF)
13149 {
13150 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13151 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13152 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13153 }
13154 else
13155 {
13156 pMixedCtx->eflags.Bits.u1RF = 0;
13157 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
13158 }
13159 break;
13160 }
13161
13162 default:
13163 {
13164 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13165 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13166 EMCODETYPE_SUPERVISOR);
13167 rc = VBOXSTRICTRC_VAL(rc2);
13168 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13169 /** @todo We have to set pending-debug exceptions here when the guest is
13170 * single-stepping depending on the instruction that was interpreted. */
13171 Log4Func(("#GP rc=%Rrc\n", rc));
13172 break;
13173 }
13174 }
13175 }
13176 else
13177 rc = VERR_EM_INTERPRETER;
13178
13179 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13180 ("#GP Unexpected rc=%Rrc\n", rc));
13181 return rc;
13182}
13183
13184
13185/**
13186 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13187 * the exception reported in the VMX transient structure back into the VM.
13188 *
13189 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13190 * up-to-date.
13191 */
13192static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13193{
13194 RT_NOREF_PV(pMixedCtx);
13195 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13196#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13197 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13198 ("uVector=%#x u32XcptBitmap=%#X32\n",
13199 VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13200#endif
13201
13202 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13203 hmR0VmxCheckExitDueToEventDelivery(). */
13204 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13205 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13206 AssertRCReturn(rc, rc);
13207 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13208
13209#ifdef DEBUG_ramshankar
13210 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS
13211 | CPUMCTX_EXTRN_RIP);
13212 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13213 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13214#endif
13215
13216 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13217 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13218 return VINF_SUCCESS;
13219}
13220
13221
13222/**
13223 * VM-exit exception handler for \#PF (Page-fault exception).
13224 */
13225static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13226{
13227 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13228 PVM pVM = pVCpu->CTX_SUFF(pVM);
13229 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13230 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13231 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13232 AssertRCReturn(rc, rc);
13233
13234 if (!pVM->hm.s.fNestedPaging)
13235 { /* likely */ }
13236 else
13237 {
13238#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13239 Assert(pVCpu->hm.s.fUsingDebugLoop);
13240#endif
13241 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13242 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13243 {
13244 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13245 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13246 }
13247 else
13248 {
13249 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13250 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13251 Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13252 }
13253 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13254 return rc;
13255 }
13256
13257 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13258 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13259 if (pVmxTransient->fVectoringPF)
13260 {
13261 Assert(pVCpu->hm.s.Event.fPending);
13262 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13263 }
13264
13265 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13266 AssertRCReturn(rc, rc);
13267
13268 Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13269 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13270
13271 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13272 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13273 (RTGCPTR)pVmxTransient->uExitQualification);
13274
13275 Log4Func(("#PF: rc=%Rrc\n", rc));
13276 if (rc == VINF_SUCCESS)
13277 {
13278 /*
13279 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13280 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13281 */
13282 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13283 TRPMResetTrap(pVCpu);
13284 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13285 return rc;
13286 }
13287
13288 if (rc == VINF_EM_RAW_GUEST_TRAP)
13289 {
13290 if (!pVmxTransient->fVectoringDoublePF)
13291 {
13292 /* It's a guest page fault and needs to be reflected to the guest. */
13293 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13294 TRPMResetTrap(pVCpu);
13295 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13296 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13297 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13298 }
13299 else
13300 {
13301 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13302 TRPMResetTrap(pVCpu);
13303 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13304 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13305 Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13306 }
13307
13308 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13309 return VINF_SUCCESS;
13310 }
13311
13312 TRPMResetTrap(pVCpu);
13313 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13314 return rc;
13315}
13316
13317/** @} */
13318
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette