VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 72805

Last change on this file since 72805 was 72805, checked in by vboxsync, 6 years ago

VMM/HM: bugref:9193 Fixes honoring pending VMCPU_FF_HM_UPDATE_CR3 and VMCPU_FF_HM_UPDATE_PAE_PDPES
before re-entering guest execution with VT-x R0 code.
Avoid a couple of VMWRITES because we already have cache's of the values (CR4 Mask, CR0 mask).
Parameter cleanup.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 571.1 KB
Line 
1/* $Id: HMVMXR0.cpp 72805 2018-07-03 04:05:43Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/x86.h>
25#include <iprt/asm-amd64-x86.h>
26#include <iprt/thread.h>
27
28#include <VBox/vmm/pdmapi.h>
29#include <VBox/vmm/dbgf.h>
30#include <VBox/vmm/iem.h>
31#include <VBox/vmm/iom.h>
32#include <VBox/vmm/selm.h>
33#include <VBox/vmm/tm.h>
34#include <VBox/vmm/gim.h>
35#include <VBox/vmm/apic.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#include "HMInternal.h"
40#include <VBox/vmm/vm.h>
41#include "HMVMXR0.h"
42#include "dtrace/VBoxVMM.h"
43
44#ifdef DEBUG_ramshankar
45# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50# define HMVMX_ALWAYS_TRAP_PF
51# define HMVMX_ALWAYS_FLUSH_TLB
52# define HMVMX_ALWAYS_SWAP_EFER
53#endif
54
55
56/*********************************************************************************************************************************
57* Defined Constants And Macros *
58*********************************************************************************************************************************/
59/** Use the function table. */
60#define HMVMX_USE_FUNCTION_TABLE
61
62/** Determine which tagged-TLB flush handler to use. */
63#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68/** @name HMVMX_READ_XXX
69 * Flags to skip redundant reads of some common VMCS fields that are not part of
70 * the guest-CPU or VCPU state but are needed while handling VM-exits.
71 */
72#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
73#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
74#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
75#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
76#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
77#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
78#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
79/** @} */
80
81/**
82 * States of the VMCS.
83 *
84 * This does not reflect all possible VMCS states but currently only those
85 * needed for maintaining the VMCS consistently even when thread-context hooks
86 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
87 */
88#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
89#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
90#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
91
92/**
93 * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
94 * guest using hardware-assisted VMX.
95 *
96 * This excludes state like GPRs (other than RSP) which are always are
97 * swapped and restored across the world-switch and also registers like EFER,
98 * MSR which cannot be modified by the guest without causing a VM-exit.
99 */
100#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
101 | CPUMCTX_EXTRN_RFLAGS \
102 | CPUMCTX_EXTRN_RSP \
103 | CPUMCTX_EXTRN_SREG_MASK \
104 | CPUMCTX_EXTRN_TABLE_MASK \
105 | CPUMCTX_EXTRN_KERNEL_GS_BASE \
106 | CPUMCTX_EXTRN_SYSCALL_MSRS \
107 | CPUMCTX_EXTRN_SYSENTER_MSRS \
108 | CPUMCTX_EXTRN_TSC_AUX \
109 | CPUMCTX_EXTRN_OTHER_MSRS \
110 | CPUMCTX_EXTRN_CR0 \
111 | CPUMCTX_EXTRN_CR3 \
112 | CPUMCTX_EXTRN_CR4 \
113 | CPUMCTX_EXTRN_DR7 \
114 | CPUMCTX_EXTRN_HM_VMX_MASK)
115
116/**
117 * Exception bitmap mask for real-mode guests (real-on-v86).
118 *
119 * We need to intercept all exceptions manually except:
120 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
121 * due to bugs in Intel CPUs.
122 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
123 * support.
124 */
125#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
126 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
127 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
128 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
129 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
130 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
131 | RT_BIT(X86_XCPT_XF))
132
133/**
134 * Exception bitmap mask for all contributory exceptions.
135 *
136 * Page fault is deliberately excluded here as it's conditional as to whether
137 * it's contributory or benign. Page faults are handled separately.
138 */
139#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
140 | RT_BIT(X86_XCPT_DE))
141
142/** Maximum VM-instruction error number. */
143#define HMVMX_INSTR_ERROR_MAX 28
144
145/** Profiling macro. */
146#ifdef HM_PROFILE_EXIT_DISPATCH
147# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
148# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
149#else
150# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
151# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
152#endif
153
154/** Assert that preemption is disabled or covered by thread-context hooks. */
155#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
156 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
157
158/** Assert that we haven't migrated CPUs when thread-context hooks are not
159 * used. */
160#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
161 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
162 ("Illegal migration! Entered on CPU %u Current %u\n", \
163 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
164
165/** Helper macro for VM-exit handlers called unexpectedly. */
166#define HMVMX_RETURN_UNEXPECTED_EXIT() \
167 do { \
168 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
169 return VERR_VMX_UNEXPECTED_EXIT; \
170 } while (0)
171
172/** Macro for saving segment registers from VMCS into the guest-CPU
173 * context. */
174#ifdef VMX_USE_CACHED_VMCS_ACCESSES
175# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
176 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
177 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
178#else
179# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
180 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
181 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
182#endif
183
184
185/*********************************************************************************************************************************
186* Structures and Typedefs *
187*********************************************************************************************************************************/
188/**
189 * VMX transient state.
190 *
191 * A state structure for holding miscellaneous information across
192 * VMX non-root operation and restored after the transition.
193 */
194typedef struct VMXTRANSIENT
195{
196 /** The host's rflags/eflags. */
197 RTCCUINTREG fEFlags;
198#if HC_ARCH_BITS == 32
199 uint32_t u32Alignment0;
200#endif
201 /** The guest's TPR value used for TPR shadowing. */
202 uint8_t u8GuestTpr;
203 /** Alignment. */
204 uint8_t abAlignment0[7];
205
206 /** The basic VM-exit reason. */
207 uint16_t uExitReason;
208 /** Alignment. */
209 uint16_t u16Alignment0;
210 /** The VM-exit interruption error code. */
211 uint32_t uExitIntErrorCode;
212 /** The VM-exit exit code qualification. */
213 uint64_t uExitQualification;
214
215 /** The VM-exit interruption-information field. */
216 uint32_t uExitIntInfo;
217 /** The VM-exit instruction-length field. */
218 uint32_t cbInstr;
219 /** The VM-exit instruction-information field. */
220 union
221 {
222 /** Plain unsigned int representation. */
223 uint32_t u;
224 /** INS and OUTS information. */
225 struct
226 {
227 uint32_t u7Reserved0 : 7;
228 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
229 uint32_t u3AddrSize : 3;
230 uint32_t u5Reserved1 : 5;
231 /** The segment register (X86_SREG_XXX). */
232 uint32_t iSegReg : 3;
233 uint32_t uReserved2 : 14;
234 } StrIo;
235 /** INVEPT, INVVPID, INVPCID information. */
236 struct
237 {
238 /** Scaling; 0=no scaling, 1=scale-by-2, 2=scale-by-4, 3=scale-by-8. */
239 uint32_t u2Scaling : 2;
240 uint32_t u5Reserved0 : 5;
241 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
242 uint32_t u3AddrSize : 3;
243 uint32_t u1Reserved0 : 1;
244 uint32_t u4Reserved0 : 4;
245 /** The segment register (X86_SREG_XXX). */
246 uint32_t iSegReg : 3;
247 /** The index register (X86_GREG_XXX). */
248 uint32_t iIdxReg : 4;
249 /** Set if index register is invalid. */
250 uint32_t fIdxRegValid : 1;
251 /** The base register (X86_GREG_XXX). */
252 uint32_t iBaseReg : 4;
253 /** Set if base register is invalid. */
254 uint32_t fBaseRegValid : 1;
255 /** Register 2 (X86_GREG_XXX). */
256 uint32_t iReg2 : 4;
257 } Inv;
258 } ExitInstrInfo;
259 /** Whether the VM-entry failed or not. */
260 bool fVMEntryFailed;
261 /** Alignment. */
262 uint8_t abAlignment1[3];
263
264 /** The VM-entry interruption-information field. */
265 uint32_t uEntryIntInfo;
266 /** The VM-entry exception error code field. */
267 uint32_t uEntryXcptErrorCode;
268 /** The VM-entry instruction length field. */
269 uint32_t cbEntryInstr;
270
271 /** IDT-vectoring information field. */
272 uint32_t uIdtVectoringInfo;
273 /** IDT-vectoring error code. */
274 uint32_t uIdtVectoringErrorCode;
275
276 /** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
277 uint32_t fVmcsFieldsRead;
278
279 /** Whether the guest debug state was active at the time of VM-exit. */
280 bool fWasGuestDebugStateActive;
281 /** Whether the hyper debug state was active at the time of VM-exit. */
282 bool fWasHyperDebugStateActive;
283 /** Whether TSC-offsetting should be setup before VM-entry. */
284 bool fUpdateTscOffsettingAndPreemptTimer;
285 /** Whether the VM-exit was caused by a page-fault during delivery of a
286 * contributory exception or a page-fault. */
287 bool fVectoringDoublePF;
288 /** Whether the VM-exit was caused by a page-fault during delivery of an
289 * external interrupt or NMI. */
290 bool fVectoringPF;
291} VMXTRANSIENT;
292AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
293AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
294AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
295AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
296AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
297/** Pointer to VMX transient state. */
298typedef VMXTRANSIENT *PVMXTRANSIENT;
299
300
301/**
302 * MSR-bitmap read permissions.
303 */
304typedef enum VMXMSREXITREAD
305{
306 /** Reading this MSR causes a VM-exit. */
307 VMXMSREXIT_INTERCEPT_READ = 0xb,
308 /** Reading this MSR does not cause a VM-exit. */
309 VMXMSREXIT_PASSTHRU_READ
310} VMXMSREXITREAD;
311/** Pointer to MSR-bitmap read permissions. */
312typedef VMXMSREXITREAD* PVMXMSREXITREAD;
313
314/**
315 * MSR-bitmap write permissions.
316 */
317typedef enum VMXMSREXITWRITE
318{
319 /** Writing to this MSR causes a VM-exit. */
320 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
321 /** Writing to this MSR does not cause a VM-exit. */
322 VMXMSREXIT_PASSTHRU_WRITE
323} VMXMSREXITWRITE;
324/** Pointer to MSR-bitmap write permissions. */
325typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
326
327
328/**
329 * VMX VM-exit handler.
330 *
331 * @returns Strict VBox status code (i.e. informational status codes too).
332 * @param pVCpu The cross context virtual CPU structure.
333 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
334 * out-of-sync. Make sure to update the required
335 * fields before using them.
336 * @param pVmxTransient Pointer to the VMX-transient structure.
337 */
338#ifndef HMVMX_USE_FUNCTION_TABLE
339typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
340#else
341typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
342/** Pointer to VM-exit handler. */
343typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
344#endif
345
346/**
347 * VMX VM-exit handler, non-strict status code.
348 *
349 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
350 *
351 * @returns VBox status code, no informational status code returned.
352 * @param pVCpu The cross context virtual CPU structure.
353 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
354 * out-of-sync. Make sure to update the required
355 * fields before using them.
356 * @param pVmxTransient Pointer to the VMX-transient structure.
357 *
358 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
359 * use of that status code will be replaced with VINF_EM_SOMETHING
360 * later when switching over to IEM.
361 */
362#ifndef HMVMX_USE_FUNCTION_TABLE
363typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
364#else
365typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
366#endif
367
368
369/*********************************************************************************************************************************
370* Internal Functions *
371*********************************************************************************************************************************/
372static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
373static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
374static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
375static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
376static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
377 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
378#if HC_ARCH_BITS == 32
379static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
380#endif
381#ifndef HMVMX_USE_FUNCTION_TABLE
382DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
383# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
384# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
385#else
386# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
387# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
388#endif
389
390
391/** @name VM-exit handlers.
392 * @{
393 */
394static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
395static FNVMXEXITHANDLER hmR0VmxExitExtInt;
396static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
397static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
398static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
399static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
400static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
401static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
402static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
403static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
404static FNVMXEXITHANDLER hmR0VmxExitCpuid;
405static FNVMXEXITHANDLER hmR0VmxExitGetsec;
406static FNVMXEXITHANDLER hmR0VmxExitHlt;
407static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
408static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
409static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
410static FNVMXEXITHANDLER hmR0VmxExitVmcall;
411static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
412static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
413static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
414static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
415static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
416static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
417static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
418static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
419static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
420static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
421static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
422static FNVMXEXITHANDLER hmR0VmxExitMwait;
423static FNVMXEXITHANDLER hmR0VmxExitMtf;
424static FNVMXEXITHANDLER hmR0VmxExitMonitor;
425static FNVMXEXITHANDLER hmR0VmxExitPause;
426static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
427static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
428static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
429static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
430static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
431static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
432static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
433static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
434static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
435static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
436static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
437static FNVMXEXITHANDLER hmR0VmxExitRdrand;
438static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
439/** @} */
440
441static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
442static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
443static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
444static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
445static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
446static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
447static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
448static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCPUMCTX pCtx);
449
450
451/*********************************************************************************************************************************
452* Global Variables *
453*********************************************************************************************************************************/
454#ifdef HMVMX_USE_FUNCTION_TABLE
455
456/**
457 * VMX_EXIT dispatch table.
458 */
459static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
460{
461 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
462 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
463 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
464 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
465 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
466 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
467 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
468 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
469 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
470 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
471 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
472 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
473 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
474 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
475 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
476 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
477 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
478 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
479 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
480 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
481 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
482 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
483 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
484 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
485 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
486 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
487 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
488 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
489 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
490 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
491 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
492 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
493 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
494 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
495 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
496 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
497 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
498 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
499 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
500 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
501 /* 40 UNDEFINED */ hmR0VmxExitPause,
502 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
503 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
504 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
505 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
506 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
507 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
508 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
509 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
510 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
511 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
512 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
513 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
514 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
515 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
516 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
517 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
518 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
519 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
520 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
521 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
522 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
523 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
524 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
525 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
526};
527#endif /* HMVMX_USE_FUNCTION_TABLE */
528
529#ifdef VBOX_STRICT
530static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
531{
532 /* 0 */ "(Not Used)",
533 /* 1 */ "VMCALL executed in VMX root operation.",
534 /* 2 */ "VMCLEAR with invalid physical address.",
535 /* 3 */ "VMCLEAR with VMXON pointer.",
536 /* 4 */ "VMLAUNCH with non-clear VMCS.",
537 /* 5 */ "VMRESUME with non-launched VMCS.",
538 /* 6 */ "VMRESUME after VMXOFF",
539 /* 7 */ "VM-entry with invalid control fields.",
540 /* 8 */ "VM-entry with invalid host state fields.",
541 /* 9 */ "VMPTRLD with invalid physical address.",
542 /* 10 */ "VMPTRLD with VMXON pointer.",
543 /* 11 */ "VMPTRLD with incorrect revision identifier.",
544 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
545 /* 13 */ "VMWRITE to read-only VMCS component.",
546 /* 14 */ "(Not Used)",
547 /* 15 */ "VMXON executed in VMX root operation.",
548 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
549 /* 17 */ "VM-entry with non-launched executing VMCS.",
550 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
551 /* 19 */ "VMCALL with non-clear VMCS.",
552 /* 20 */ "VMCALL with invalid VM-exit control fields.",
553 /* 21 */ "(Not Used)",
554 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
555 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
556 /* 24 */ "VMCALL with invalid SMM-monitor features.",
557 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
558 /* 26 */ "VM-entry with events blocked by MOV SS.",
559 /* 27 */ "(Not Used)",
560 /* 28 */ "Invalid operand to INVEPT/INVVPID."
561};
562#endif /* VBOX_STRICT */
563
564
565
566/**
567 * Updates the VM's last error record.
568 *
569 * If there was a VMX instruction error, reads the error data from the VMCS and
570 * updates VCPU's last error record as well.
571 *
572 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
573 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
574 * VERR_VMX_INVALID_VMCS_FIELD.
575 * @param rc The error code.
576 */
577static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
578{
579 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
580 || rc == VERR_VMX_UNABLE_TO_START_VM)
581 {
582 AssertPtrReturnVoid(pVCpu);
583 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
584 }
585 pVCpu->CTX_SUFF(pVM)->hm.s.lLastError = rc;
586}
587
588
589/**
590 * Reads the VM-entry interruption-information field from the VMCS into the VMX
591 * transient structure.
592 *
593 * @returns VBox status code.
594 * @param pVmxTransient Pointer to the VMX transient structure.
595 *
596 * @remarks No-long-jump zone!!!
597 */
598DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
599{
600 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
601 AssertRCReturn(rc, rc);
602 return VINF_SUCCESS;
603}
604
605#ifdef VBOX_STRICT
606/**
607 * Reads the VM-entry exception error code field from the VMCS into
608 * the VMX transient structure.
609 *
610 * @returns VBox status code.
611 * @param pVmxTransient Pointer to the VMX transient structure.
612 *
613 * @remarks No-long-jump zone!!!
614 */
615DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
616{
617 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
618 AssertRCReturn(rc, rc);
619 return VINF_SUCCESS;
620}
621
622
623/**
624 * Reads the VM-entry exception error code field from the VMCS into
625 * the VMX transient structure.
626 *
627 * @returns VBox status code.
628 * @param pVmxTransient Pointer to the VMX transient structure.
629 *
630 * @remarks No-long-jump zone!!!
631 */
632DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
633{
634 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
635 AssertRCReturn(rc, rc);
636 return VINF_SUCCESS;
637}
638#endif /* VBOX_STRICT */
639
640
641/**
642 * Reads the VM-exit interruption-information field from the VMCS into the VMX
643 * transient structure.
644 *
645 * @returns VBox status code.
646 * @param pVmxTransient Pointer to the VMX transient structure.
647 */
648DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
649{
650 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
651 {
652 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
653 AssertRCReturn(rc,rc);
654 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_INFO;
655 }
656 return VINF_SUCCESS;
657}
658
659
660/**
661 * Reads the VM-exit interruption error code from the VMCS into the VMX
662 * transient structure.
663 *
664 * @returns VBox status code.
665 * @param pVmxTransient Pointer to the VMX transient structure.
666 */
667DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
668{
669 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
670 {
671 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
672 AssertRCReturn(rc, rc);
673 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
674 }
675 return VINF_SUCCESS;
676}
677
678
679/**
680 * Reads the VM-exit instruction length field from the VMCS into the VMX
681 * transient structure.
682 *
683 * @returns VBox status code.
684 * @param pVmxTransient Pointer to the VMX transient structure.
685 */
686DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
687{
688 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
689 {
690 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
691 AssertRCReturn(rc, rc);
692 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_LEN;
693 }
694 return VINF_SUCCESS;
695}
696
697
698/**
699 * Reads the VM-exit instruction-information field from the VMCS into
700 * the VMX transient structure.
701 *
702 * @returns VBox status code.
703 * @param pVmxTransient Pointer to the VMX transient structure.
704 */
705DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
706{
707 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
708 {
709 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
710 AssertRCReturn(rc, rc);
711 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_INFO;
712 }
713 return VINF_SUCCESS;
714}
715
716
717/**
718 * Reads the exit code qualification from the VMCS into the VMX transient
719 * structure.
720 *
721 * @returns VBox status code.
722 * @param pVCpu The cross context virtual CPU structure of the
723 * calling EMT. (Required for the VMCS cache case.)
724 * @param pVmxTransient Pointer to the VMX transient structure.
725 */
726DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
727{
728 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
729 {
730 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
731 AssertRCReturn(rc, rc);
732 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION;
733 }
734 return VINF_SUCCESS;
735}
736
737
738/**
739 * Reads the IDT-vectoring information field from the VMCS into the VMX
740 * transient structure.
741 *
742 * @returns VBox status code.
743 * @param pVmxTransient Pointer to the VMX transient structure.
744 *
745 * @remarks No-long-jump zone!!!
746 */
747DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
748{
749 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
750 {
751 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
752 AssertRCReturn(rc, rc);
753 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_INFO;
754 }
755 return VINF_SUCCESS;
756}
757
758
759/**
760 * Reads the IDT-vectoring error code from the VMCS into the VMX
761 * transient structure.
762 *
763 * @returns VBox status code.
764 * @param pVmxTransient Pointer to the VMX transient structure.
765 */
766DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
767{
768 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
769 {
770 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
771 AssertRCReturn(rc, rc);
772 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
773 }
774 return VINF_SUCCESS;
775}
776
777
778/**
779 * Enters VMX root mode operation on the current CPU.
780 *
781 * @returns VBox status code.
782 * @param pVM The cross context VM structure. Can be
783 * NULL, after a resume.
784 * @param HCPhysCpuPage Physical address of the VMXON region.
785 * @param pvCpuPage Pointer to the VMXON region.
786 */
787static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
788{
789 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
790 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
791 Assert(pvCpuPage);
792 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
793
794 if (pVM)
795 {
796 /* Write the VMCS revision dword to the VMXON region. */
797 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
798 }
799
800 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
801 RTCCUINTREG fEFlags = ASMIntDisableFlags();
802
803 /* Enable the VMX bit in CR4 if necessary. */
804 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
805
806 /* Enter VMX root mode. */
807 int rc = VMXEnable(HCPhysCpuPage);
808 if (RT_FAILURE(rc))
809 {
810 if (!(uOldCr4 & X86_CR4_VMXE))
811 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
812
813 if (pVM)
814 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
815 }
816
817 /* Restore interrupts. */
818 ASMSetFlags(fEFlags);
819 return rc;
820}
821
822
823/**
824 * Exits VMX root mode operation on the current CPU.
825 *
826 * @returns VBox status code.
827 */
828static int hmR0VmxLeaveRootMode(void)
829{
830 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
831
832 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
833 RTCCUINTREG fEFlags = ASMIntDisableFlags();
834
835 /* If we're for some reason not in VMX root mode, then don't leave it. */
836 RTCCUINTREG uHostCR4 = ASMGetCR4();
837
838 int rc;
839 if (uHostCR4 & X86_CR4_VMXE)
840 {
841 /* Exit VMX root mode and clear the VMX bit in CR4. */
842 VMXDisable();
843 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
844 rc = VINF_SUCCESS;
845 }
846 else
847 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
848
849 /* Restore interrupts. */
850 ASMSetFlags(fEFlags);
851 return rc;
852}
853
854
855/**
856 * Allocates and maps one physically contiguous page. The allocated page is
857 * zero'd out. (Used by various VT-x structures).
858 *
859 * @returns IPRT status code.
860 * @param pMemObj Pointer to the ring-0 memory object.
861 * @param ppVirt Where to store the virtual address of the
862 * allocation.
863 * @param pHCPhys Where to store the physical address of the
864 * allocation.
865 */
866static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
867{
868 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
869 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
870 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
871
872 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
873 if (RT_FAILURE(rc))
874 return rc;
875 *ppVirt = RTR0MemObjAddress(*pMemObj);
876 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
877 ASMMemZero32(*ppVirt, PAGE_SIZE);
878 return VINF_SUCCESS;
879}
880
881
882/**
883 * Frees and unmaps an allocated physical page.
884 *
885 * @param pMemObj Pointer to the ring-0 memory object.
886 * @param ppVirt Where to re-initialize the virtual address of
887 * allocation as 0.
888 * @param pHCPhys Where to re-initialize the physical address of the
889 * allocation as 0.
890 */
891static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
892{
893 AssertPtr(pMemObj);
894 AssertPtr(ppVirt);
895 AssertPtr(pHCPhys);
896 if (*pMemObj != NIL_RTR0MEMOBJ)
897 {
898 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
899 AssertRC(rc);
900 *pMemObj = NIL_RTR0MEMOBJ;
901 *ppVirt = 0;
902 *pHCPhys = 0;
903 }
904}
905
906
907/**
908 * Worker function to free VT-x related structures.
909 *
910 * @returns IPRT status code.
911 * @param pVM The cross context VM structure.
912 */
913static void hmR0VmxStructsFree(PVM pVM)
914{
915 for (VMCPUID i = 0; i < pVM->cCpus; i++)
916 {
917 PVMCPU pVCpu = &pVM->aCpus[i];
918 AssertPtr(pVCpu);
919
920 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
921 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
922
923 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
924 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
925
926 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
927 }
928
929 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
930#ifdef VBOX_WITH_CRASHDUMP_MAGIC
931 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
932#endif
933}
934
935
936/**
937 * Worker function to allocate VT-x related VM structures.
938 *
939 * @returns IPRT status code.
940 * @param pVM The cross context VM structure.
941 */
942static int hmR0VmxStructsAlloc(PVM pVM)
943{
944 /*
945 * Initialize members up-front so we can cleanup properly on allocation failure.
946 */
947#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
948 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
949 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
950 pVM->hm.s.vmx.HCPhys##a_Name = 0;
951
952#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
953 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
954 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
955 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
956
957#ifdef VBOX_WITH_CRASHDUMP_MAGIC
958 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
959#endif
960 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
961
962 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
963 for (VMCPUID i = 0; i < pVM->cCpus; i++)
964 {
965 PVMCPU pVCpu = &pVM->aCpus[i];
966 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
967 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
968 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
969 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
970 }
971#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
972#undef VMXLOCAL_INIT_VM_MEMOBJ
973
974 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
975 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
976 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
977 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
978
979 /*
980 * Allocate all the VT-x structures.
981 */
982 int rc = VINF_SUCCESS;
983#ifdef VBOX_WITH_CRASHDUMP_MAGIC
984 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
985 if (RT_FAILURE(rc))
986 goto cleanup;
987 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
988 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
989#endif
990
991 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
992 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
993 {
994 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
995 &pVM->hm.s.vmx.HCPhysApicAccess);
996 if (RT_FAILURE(rc))
997 goto cleanup;
998 }
999
1000 /*
1001 * Initialize per-VCPU VT-x structures.
1002 */
1003 for (VMCPUID i = 0; i < pVM->cCpus; i++)
1004 {
1005 PVMCPU pVCpu = &pVM->aCpus[i];
1006 AssertPtr(pVCpu);
1007
1008 /* Allocate the VM control structure (VMCS). */
1009 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1010 if (RT_FAILURE(rc))
1011 goto cleanup;
1012
1013 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1014 if ( PDMHasApic(pVM)
1015 && (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW))
1016 {
1017 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1018 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1019 if (RT_FAILURE(rc))
1020 goto cleanup;
1021 }
1022
1023 /*
1024 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1025 * transparent accesses of specific MSRs.
1026 *
1027 * If the condition for enabling MSR bitmaps changes here, don't forget to
1028 * update HMAreMsrBitmapsAvailable().
1029 */
1030 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1031 {
1032 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1033 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1034 if (RT_FAILURE(rc))
1035 goto cleanup;
1036 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1037 }
1038
1039 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1040 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1041 if (RT_FAILURE(rc))
1042 goto cleanup;
1043
1044 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1045 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1046 if (RT_FAILURE(rc))
1047 goto cleanup;
1048 }
1049
1050 return VINF_SUCCESS;
1051
1052cleanup:
1053 hmR0VmxStructsFree(pVM);
1054 return rc;
1055}
1056
1057
1058/**
1059 * Does global VT-x initialization (called during module initialization).
1060 *
1061 * @returns VBox status code.
1062 */
1063VMMR0DECL(int) VMXR0GlobalInit(void)
1064{
1065#ifdef HMVMX_USE_FUNCTION_TABLE
1066 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1067# ifdef VBOX_STRICT
1068 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1069 Assert(g_apfnVMExitHandlers[i]);
1070# endif
1071#endif
1072 return VINF_SUCCESS;
1073}
1074
1075
1076/**
1077 * Does global VT-x termination (called during module termination).
1078 */
1079VMMR0DECL(void) VMXR0GlobalTerm()
1080{
1081 /* Nothing to do currently. */
1082}
1083
1084
1085/**
1086 * Sets up and activates VT-x on the current CPU.
1087 *
1088 * @returns VBox status code.
1089 * @param pHostCpu Pointer to the global CPU info struct.
1090 * @param pVM The cross context VM structure. Can be
1091 * NULL after a host resume operation.
1092 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1093 * fEnabledByHost is @c true).
1094 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1095 * @a fEnabledByHost is @c true).
1096 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1097 * enable VT-x on the host.
1098 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1099 */
1100VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1101 void *pvMsrs)
1102{
1103 Assert(pHostCpu);
1104 Assert(pvMsrs);
1105 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1106
1107 /* Enable VT-x if it's not already enabled by the host. */
1108 if (!fEnabledByHost)
1109 {
1110 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1111 if (RT_FAILURE(rc))
1112 return rc;
1113 }
1114
1115 /*
1116 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1117 * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1118 * invalidated when flushing by VPID.
1119 */
1120 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1121 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1122 {
1123 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1124 pHostCpu->fFlushAsidBeforeUse = false;
1125 }
1126 else
1127 pHostCpu->fFlushAsidBeforeUse = true;
1128
1129 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1130 ++pHostCpu->cTlbFlushes;
1131
1132 return VINF_SUCCESS;
1133}
1134
1135
1136/**
1137 * Deactivates VT-x on the current CPU.
1138 *
1139 * @returns VBox status code.
1140 * @param pHostCpu Pointer to the global CPU info struct.
1141 * @param pvCpuPage Pointer to the VMXON region.
1142 * @param HCPhysCpuPage Physical address of the VMXON region.
1143 *
1144 * @remarks This function should never be called when SUPR0EnableVTx() or
1145 * similar was used to enable VT-x on the host.
1146 */
1147VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1148{
1149 RT_NOREF3(pHostCpu, pvCpuPage, HCPhysCpuPage);
1150
1151 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1152 return hmR0VmxLeaveRootMode();
1153}
1154
1155
1156/**
1157 * Sets the permission bits for the specified MSR in the MSR bitmap.
1158 *
1159 * @param pVCpu The cross context virtual CPU structure.
1160 * @param uMsr The MSR value.
1161 * @param enmRead Whether reading this MSR causes a VM-exit.
1162 * @param enmWrite Whether writing this MSR causes a VM-exit.
1163 */
1164static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1165{
1166 int32_t iBit;
1167 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1168
1169 /*
1170 * Layout:
1171 * 0x000 - 0x3ff - Low MSR read bits
1172 * 0x400 - 0x7ff - High MSR read bits
1173 * 0x800 - 0xbff - Low MSR write bits
1174 * 0xc00 - 0xfff - High MSR write bits
1175 */
1176 if (uMsr <= 0x00001FFF)
1177 iBit = uMsr;
1178 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1179 {
1180 iBit = uMsr - UINT32_C(0xC0000000);
1181 pbMsrBitmap += 0x400;
1182 }
1183 else
1184 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1185
1186 Assert(iBit <= 0x1fff);
1187 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1188 ASMBitSet(pbMsrBitmap, iBit);
1189 else
1190 ASMBitClear(pbMsrBitmap, iBit);
1191
1192 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1193 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1194 else
1195 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1196}
1197
1198
1199#ifdef VBOX_STRICT
1200/**
1201 * Gets the permission bits for the specified MSR in the MSR bitmap.
1202 *
1203 * @returns VBox status code.
1204 * @retval VINF_SUCCESS if the specified MSR is found.
1205 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1206 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1207 *
1208 * @param pVCpu The cross context virtual CPU structure.
1209 * @param uMsr The MSR.
1210 * @param penmRead Where to store the read permissions.
1211 * @param penmWrite Where to store the write permissions.
1212 */
1213static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1214{
1215 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1216 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1217 int32_t iBit;
1218 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1219
1220 /* See hmR0VmxSetMsrPermission() for the layout. */
1221 if (uMsr <= 0x00001FFF)
1222 iBit = uMsr;
1223 else if ( uMsr >= 0xC0000000
1224 && uMsr <= 0xC0001FFF)
1225 {
1226 iBit = (uMsr - 0xC0000000);
1227 pbMsrBitmap += 0x400;
1228 }
1229 else
1230 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1231
1232 Assert(iBit <= 0x1fff);
1233 if (ASMBitTest(pbMsrBitmap, iBit))
1234 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1235 else
1236 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1237
1238 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1239 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1240 else
1241 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1242 return VINF_SUCCESS;
1243}
1244#endif /* VBOX_STRICT */
1245
1246
1247/**
1248 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1249 * area.
1250 *
1251 * @returns VBox status code.
1252 * @param pVCpu The cross context virtual CPU structure.
1253 * @param cMsrs The number of MSRs.
1254 */
1255static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1256{
1257 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1258 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1259 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1260 {
1261 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1262 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1263 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1264 }
1265
1266 /* Update number of guest MSRs to load/store across the world-switch. */
1267 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1268 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1269
1270 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1271 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1272 AssertRCReturn(rc, rc);
1273
1274 /* Update the VCPU's copy of the MSR count. */
1275 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1276
1277 return VINF_SUCCESS;
1278}
1279
1280
1281/**
1282 * Adds a new (or updates the value of an existing) guest/host MSR
1283 * pair to be swapped during the world-switch as part of the
1284 * auto-load/store MSR area in the VMCS.
1285 *
1286 * @returns VBox status code.
1287 * @param pVCpu The cross context virtual CPU structure.
1288 * @param uMsr The MSR.
1289 * @param uGuestMsrValue Value of the guest MSR.
1290 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1291 * necessary.
1292 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1293 * its value was updated. Optional, can be NULL.
1294 */
1295static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1296 bool *pfAddedAndUpdated)
1297{
1298 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1299 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1300 uint32_t i;
1301 for (i = 0; i < cMsrs; i++)
1302 {
1303 if (pGuestMsr->u32Msr == uMsr)
1304 break;
1305 pGuestMsr++;
1306 }
1307
1308 bool fAdded = false;
1309 if (i == cMsrs)
1310 {
1311 ++cMsrs;
1312 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1313 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1314
1315 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1316 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1317 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1318
1319 fAdded = true;
1320 }
1321
1322 /* Update the MSR values in the auto-load/store MSR area. */
1323 pGuestMsr->u32Msr = uMsr;
1324 pGuestMsr->u64Value = uGuestMsrValue;
1325
1326 /* Create/update the MSR slot in the host MSR area. */
1327 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1328 pHostMsr += i;
1329 pHostMsr->u32Msr = uMsr;
1330
1331 /*
1332 * Update the host MSR only when requested by the caller AND when we're
1333 * adding it to the auto-load/store area. Otherwise, it would have been
1334 * updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1335 */
1336 bool fUpdatedMsrValue = false;
1337 if ( fAdded
1338 && fUpdateHostMsr)
1339 {
1340 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1341 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1342 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1343 fUpdatedMsrValue = true;
1344 }
1345
1346 if (pfAddedAndUpdated)
1347 *pfAddedAndUpdated = fUpdatedMsrValue;
1348 return VINF_SUCCESS;
1349}
1350
1351
1352/**
1353 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1354 * auto-load/store MSR area in the VMCS.
1355 *
1356 * @returns VBox status code.
1357 * @param pVCpu The cross context virtual CPU structure.
1358 * @param uMsr The MSR.
1359 */
1360static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1361{
1362 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1363 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1364 for (uint32_t i = 0; i < cMsrs; i++)
1365 {
1366 /* Find the MSR. */
1367 if (pGuestMsr->u32Msr == uMsr)
1368 {
1369 /* If it's the last MSR, simply reduce the count. */
1370 if (i == cMsrs - 1)
1371 {
1372 --cMsrs;
1373 break;
1374 }
1375
1376 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1377 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1378 pLastGuestMsr += cMsrs - 1;
1379 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1380 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1381
1382 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1383 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1384 pLastHostMsr += cMsrs - 1;
1385 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1386 pHostMsr->u64Value = pLastHostMsr->u64Value;
1387 --cMsrs;
1388 break;
1389 }
1390 pGuestMsr++;
1391 }
1392
1393 /* Update the VMCS if the count changed (meaning the MSR was found). */
1394 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1395 {
1396 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1397 AssertRCReturn(rc, rc);
1398
1399 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1400 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1401 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1402
1403 Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1404 return VINF_SUCCESS;
1405 }
1406
1407 return VERR_NOT_FOUND;
1408}
1409
1410
1411/**
1412 * Checks if the specified guest MSR is part of the auto-load/store area in
1413 * the VMCS.
1414 *
1415 * @returns true if found, false otherwise.
1416 * @param pVCpu The cross context virtual CPU structure.
1417 * @param uMsr The MSR to find.
1418 */
1419static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1420{
1421 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1422 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1423
1424 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1425 {
1426 if (pGuestMsr->u32Msr == uMsr)
1427 return true;
1428 }
1429 return false;
1430}
1431
1432
1433/**
1434 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1435 *
1436 * @param pVCpu The cross context virtual CPU structure.
1437 *
1438 * @remarks No-long-jump zone!!!
1439 */
1440static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1441{
1442 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1443 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1444 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1445 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1446
1447 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1448 {
1449 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1450
1451 /*
1452 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1453 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1454 */
1455 if (pHostMsr->u32Msr == MSR_K6_EFER)
1456 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1457 else
1458 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1459 }
1460
1461 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1462}
1463
1464
1465/**
1466 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1467 * perform lazy restoration of the host MSRs while leaving VT-x.
1468 *
1469 * @param pVCpu The cross context virtual CPU structure.
1470 *
1471 * @remarks No-long-jump zone!!!
1472 */
1473static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1474{
1475 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1476
1477 /*
1478 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1479 */
1480 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1481 {
1482 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1483#if HC_ARCH_BITS == 64
1484 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1485 {
1486 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1487 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1488 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1489 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1490 }
1491#endif
1492 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1493 }
1494}
1495
1496
1497/**
1498 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1499 * lazily while leaving VT-x.
1500 *
1501 * @returns true if it does, false otherwise.
1502 * @param pVCpu The cross context virtual CPU structure.
1503 * @param uMsr The MSR to check.
1504 */
1505static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1506{
1507 NOREF(pVCpu);
1508#if HC_ARCH_BITS == 64
1509 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1510 {
1511 switch (uMsr)
1512 {
1513 case MSR_K8_LSTAR:
1514 case MSR_K6_STAR:
1515 case MSR_K8_SF_MASK:
1516 case MSR_K8_KERNEL_GS_BASE:
1517 return true;
1518 }
1519 }
1520#else
1521 RT_NOREF(pVCpu, uMsr);
1522#endif
1523 return false;
1524}
1525
1526
1527/**
1528 * Loads a set of guests MSRs to allow read/passthru to the guest.
1529 *
1530 * The name of this function is slightly confusing. This function does NOT
1531 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1532 * common prefix for functions dealing with "lazy restoration" of the shared
1533 * MSRs.
1534 *
1535 * @param pVCpu The cross context virtual CPU structure.
1536 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1537 * out-of-sync. Make sure to update the required fields
1538 * before using them.
1539 *
1540 * @remarks No-long-jump zone!!!
1541 */
1542static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1543{
1544 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1545 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1546
1547 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1548#if HC_ARCH_BITS == 64
1549 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1550 {
1551 /*
1552 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1553 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1554 * we can skip a few MSR writes.
1555 *
1556 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1557 * guest MSR values in the guest-CPU context might be different to what's currently
1558 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1559 * CPU, see @bugref{8728}.
1560 */
1561 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1562 && pMixedCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1563 && pMixedCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1564 && pMixedCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1565 && pMixedCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1566 {
1567#ifdef VBOX_STRICT
1568 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pMixedCtx->msrKERNELGSBASE);
1569 Assert(ASMRdMsr(MSR_K8_LSTAR) == pMixedCtx->msrLSTAR);
1570 Assert(ASMRdMsr(MSR_K6_STAR) == pMixedCtx->msrSTAR);
1571 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pMixedCtx->msrSFMASK);
1572#endif
1573 }
1574 else
1575 {
1576 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1577 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1578 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1579 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1580 }
1581 }
1582#else
1583 RT_NOREF(pMixedCtx);
1584#endif
1585 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1586}
1587
1588
1589/**
1590 * Performs lazy restoration of the set of host MSRs if they were previously
1591 * loaded with guest MSR values.
1592 *
1593 * @param pVCpu The cross context virtual CPU structure.
1594 *
1595 * @remarks No-long-jump zone!!!
1596 * @remarks The guest MSRs should have been saved back into the guest-CPU
1597 * context by hmR0VmxImportGuestState()!!!
1598 */
1599static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1600{
1601 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1602 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1603
1604 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1605 {
1606 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1607#if HC_ARCH_BITS == 64
1608 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1609 {
1610 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1611 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1612 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1613 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1614 }
1615#endif
1616 }
1617 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1618}
1619
1620
1621/**
1622 * Verifies that our cached values of the VMCS controls are all
1623 * consistent with what's actually present in the VMCS.
1624 *
1625 * @returns VBox status code.
1626 * @param pVCpu The cross context virtual CPU structure.
1627 */
1628static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1629{
1630 uint32_t u32Val;
1631 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1632 AssertRCReturn(rc, rc);
1633 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1634 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1635
1636 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1637 AssertRCReturn(rc, rc);
1638 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1639 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1640
1641 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1642 AssertRCReturn(rc, rc);
1643 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1644 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1645
1646 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1647 AssertRCReturn(rc, rc);
1648 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1649 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1650
1651 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1652 {
1653 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1654 AssertRCReturn(rc, rc);
1655 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1656 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1657 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1658 }
1659
1660 return VINF_SUCCESS;
1661}
1662
1663
1664#ifdef VBOX_STRICT
1665/**
1666 * Verifies that our cached host EFER value has not changed
1667 * since we cached it.
1668 *
1669 * @param pVCpu The cross context virtual CPU structure.
1670 */
1671static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1672{
1673 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1674
1675 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1676 {
1677 uint64_t u64Val;
1678 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1679 AssertRC(rc);
1680
1681 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1682 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1683 }
1684}
1685
1686
1687/**
1688 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1689 * VMCS are correct.
1690 *
1691 * @param pVCpu The cross context virtual CPU structure.
1692 */
1693static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1694{
1695 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1696
1697 /* Verify MSR counts in the VMCS are what we think it should be. */
1698 uint32_t cMsrs;
1699 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1700 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1701
1702 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1703 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1704
1705 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1706 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1707
1708 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1709 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1710 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1711 {
1712 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1713 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1714 pGuestMsr->u32Msr, cMsrs));
1715
1716 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1717 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1718 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1719
1720 /* Verify that the permissions are as expected in the MSR bitmap. */
1721 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1722 {
1723 VMXMSREXITREAD enmRead;
1724 VMXMSREXITWRITE enmWrite;
1725 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1726 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1727 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1728 {
1729 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1730 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1731 }
1732 else
1733 {
1734 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1735 pGuestMsr->u32Msr, cMsrs));
1736 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1737 pGuestMsr->u32Msr, cMsrs));
1738 }
1739 }
1740 }
1741}
1742#endif /* VBOX_STRICT */
1743
1744
1745/**
1746 * Flushes the TLB using EPT.
1747 *
1748 * @returns VBox status code.
1749 * @param pVCpu The cross context virtual CPU structure of the calling
1750 * EMT. Can be NULL depending on @a enmFlush.
1751 * @param enmFlush Type of flush.
1752 *
1753 * @remarks Caller is responsible for making sure this function is called only
1754 * when NestedPaging is supported and providing @a enmFlush that is
1755 * supported by the CPU.
1756 * @remarks Can be called with interrupts disabled.
1757 */
1758static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1759{
1760 uint64_t au64Descriptor[2];
1761 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1762 au64Descriptor[0] = 0;
1763 else
1764 {
1765 Assert(pVCpu);
1766 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1767 }
1768 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1769
1770 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1771 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1772 rc));
1773 if ( RT_SUCCESS(rc)
1774 && pVCpu)
1775 {
1776 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1777 }
1778}
1779
1780
1781/**
1782 * Flushes the TLB using VPID.
1783 *
1784 * @returns VBox status code.
1785 * @param pVCpu The cross context virtual CPU structure of the calling
1786 * EMT. Can be NULL depending on @a enmFlush.
1787 * @param enmFlush Type of flush.
1788 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1789 * on @a enmFlush).
1790 *
1791 * @remarks Can be called with interrupts disabled.
1792 */
1793static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1794{
1795 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
1796
1797 uint64_t au64Descriptor[2];
1798 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1799 {
1800 au64Descriptor[0] = 0;
1801 au64Descriptor[1] = 0;
1802 }
1803 else
1804 {
1805 AssertPtr(pVCpu);
1806 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1807 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1808 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1809 au64Descriptor[1] = GCPtr;
1810 }
1811
1812 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]);
1813 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmFlush,
1814 pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1815 if ( RT_SUCCESS(rc)
1816 && pVCpu)
1817 {
1818 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1819 }
1820 NOREF(rc);
1821}
1822
1823
1824/**
1825 * Invalidates a guest page by guest virtual address. Only relevant for
1826 * EPT/VPID, otherwise there is nothing really to invalidate.
1827 *
1828 * @returns VBox status code.
1829 * @param pVCpu The cross context virtual CPU structure.
1830 * @param GCVirt Guest virtual address of the page to invalidate.
1831 */
1832VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
1833{
1834 AssertPtr(pVCpu);
1835 LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
1836
1837 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1838 if (!fFlushPending)
1839 {
1840 /*
1841 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1842 * the EPT case. See @bugref{6043} and @bugref{6177}.
1843 *
1844 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1845 * as this function maybe called in a loop with individual addresses.
1846 */
1847 PVM pVM = pVCpu->CTX_SUFF(pVM);
1848 if (pVM->hm.s.vmx.fVpid)
1849 {
1850 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1851
1852#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1853 /*
1854 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1855 * where executing INVVPID outside 64-bit mode does not flush translations of
1856 * 64-bit linear addresses, see @bugref{6208#c72}.
1857 */
1858 if (RT_HI_U32(GCVirt))
1859 fVpidFlush = false;
1860#endif
1861
1862 if (fVpidFlush)
1863 {
1864 hmR0VmxFlushVpid(pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1865 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1866 }
1867 else
1868 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1869 }
1870 else if (pVM->hm.s.fNestedPaging)
1871 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1872 }
1873
1874 return VINF_SUCCESS;
1875}
1876
1877
1878/**
1879 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1880 * case where neither EPT nor VPID is supported by the CPU.
1881 *
1882 * @param pVCpu The cross context virtual CPU structure.
1883 * @param pCpu Pointer to the global HM struct.
1884 *
1885 * @remarks Called with interrupts disabled.
1886 */
1887static void hmR0VmxFlushTaggedTlbNone(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1888{
1889 AssertPtr(pVCpu);
1890 AssertPtr(pCpu);
1891
1892 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1893
1894 Assert(pCpu->idCpu != NIL_RTCPUID);
1895 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1896 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1897 pVCpu->hm.s.fForceTLBFlush = false;
1898 return;
1899}
1900
1901
1902/**
1903 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1904 *
1905 * @param pVCpu The cross context virtual CPU structure.
1906 * @param pCpu Pointer to the global HM CPU struct.
1907 *
1908 * @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
1909 * nomenclature. The reason is, to avoid confusion in compare statements
1910 * since the host-CPU copies are named "ASID".
1911 *
1912 * @remarks Called with interrupts disabled.
1913 */
1914static void hmR0VmxFlushTaggedTlbBoth(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1915{
1916#ifdef VBOX_WITH_STATISTICS
1917 bool fTlbFlushed = false;
1918# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1919# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1920 if (!fTlbFlushed) \
1921 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1922 } while (0)
1923#else
1924# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1925# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1926#endif
1927
1928 AssertPtr(pCpu);
1929 AssertPtr(pVCpu);
1930 Assert(pCpu->idCpu != NIL_RTCPUID);
1931
1932 PVM pVM = pVCpu->CTX_SUFF(pVM);
1933 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1934 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1935 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1936
1937 /*
1938 * Force a TLB flush for the first world-switch if the current CPU differs from the one we
1939 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1940 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1941 * cannot reuse the current ASID anymore.
1942 */
1943 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1944 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1945 {
1946 ++pCpu->uCurrentAsid;
1947 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1948 {
1949 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1950 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1951 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1952 }
1953
1954 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1955 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1956 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1957
1958 /*
1959 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1960 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1961 */
1962 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1963 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1964 HMVMX_SET_TAGGED_TLB_FLUSHED();
1965 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1966 }
1967
1968 /* Check for explicit TLB flushes. */
1969 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1970 {
1971 /*
1972 * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
1973 * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
1974 * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
1975 * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
1976 * mappings, see @bugref{6568}.
1977 *
1978 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
1979 */
1980 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1981 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1982 HMVMX_SET_TAGGED_TLB_FLUSHED();
1983 }
1984
1985 pVCpu->hm.s.fForceTLBFlush = false;
1986 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1987
1988 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1989 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1990 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1991 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1992 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1993 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
1994 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
1995 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1996 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1997
1998 /* Update VMCS with the VPID. */
1999 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2000 AssertRC(rc);
2001
2002#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2003}
2004
2005
2006/**
2007 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2008 *
2009 * @returns VBox status code.
2010 * @param pVCpu The cross context virtual CPU structure.
2011 * @param pCpu Pointer to the global HM CPU struct.
2012 *
2013 * @remarks Called with interrupts disabled.
2014 */
2015static void hmR0VmxFlushTaggedTlbEpt(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2016{
2017 AssertPtr(pVCpu);
2018 AssertPtr(pCpu);
2019 Assert(pCpu->idCpu != NIL_RTCPUID);
2020 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2021 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2022
2023 /*
2024 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2025 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2026 */
2027 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2028 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2029 {
2030 pVCpu->hm.s.fForceTLBFlush = true;
2031 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2032 }
2033
2034 /* Check for explicit TLB flushes. */
2035 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2036 {
2037 pVCpu->hm.s.fForceTLBFlush = true;
2038 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2039 }
2040
2041 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2042 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2043
2044 if (pVCpu->hm.s.fForceTLBFlush)
2045 {
2046 hmR0VmxFlushEpt(pVCpu, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmFlushEpt);
2047 pVCpu->hm.s.fForceTLBFlush = false;
2048 }
2049}
2050
2051
2052/**
2053 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2054 *
2055 * @returns VBox status code.
2056 * @param pVCpu The cross context virtual CPU structure.
2057 * @param pCpu Pointer to the global HM CPU struct.
2058 *
2059 * @remarks Called with interrupts disabled.
2060 */
2061static void hmR0VmxFlushTaggedTlbVpid(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2062{
2063 AssertPtr(pVCpu);
2064 AssertPtr(pCpu);
2065 Assert(pCpu->idCpu != NIL_RTCPUID);
2066 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2067 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2068
2069 /*
2070 * Force a TLB flush for the first world switch if the current CPU differs from the one we
2071 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2072 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2073 * cannot reuse the current ASID anymore.
2074 */
2075 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2076 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2077 {
2078 pVCpu->hm.s.fForceTLBFlush = true;
2079 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2080 }
2081
2082 /* Check for explicit TLB flushes. */
2083 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2084 {
2085 /*
2086 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2087 * hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2088 * fExplicitFlush = true here and change the pCpu->fFlushAsidBeforeUse check below to
2089 * include fExplicitFlush's too) - an obscure corner case.
2090 */
2091 pVCpu->hm.s.fForceTLBFlush = true;
2092 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2093 }
2094
2095 PVM pVM = pVCpu->CTX_SUFF(pVM);
2096 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2097 if (pVCpu->hm.s.fForceTLBFlush)
2098 {
2099 ++pCpu->uCurrentAsid;
2100 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2101 {
2102 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2103 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2104 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2105 }
2106
2107 pVCpu->hm.s.fForceTLBFlush = false;
2108 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2109 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2110 if (pCpu->fFlushAsidBeforeUse)
2111 {
2112 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2113 hmR0VmxFlushVpid(pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2114 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2115 {
2116 hmR0VmxFlushVpid(pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2117 pCpu->fFlushAsidBeforeUse = false;
2118 }
2119 else
2120 {
2121 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2122 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2123 }
2124 }
2125 }
2126
2127 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2128 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2129 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2130 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2131 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2132 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2133 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2134
2135 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2136 AssertRC(rc);
2137}
2138
2139
2140/**
2141 * Flushes the guest TLB entry based on CPU capabilities.
2142 *
2143 * @param pVCpu The cross context virtual CPU structure.
2144 * @param pCpu Pointer to the global HM CPU struct.
2145 */
2146DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2147{
2148#ifdef HMVMX_ALWAYS_FLUSH_TLB
2149 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2150#endif
2151 PVM pVM = pVCpu->CTX_SUFF(pVM);
2152 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2153 {
2154 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVCpu, pCpu); break;
2155 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVCpu, pCpu); break;
2156 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVCpu, pCpu); break;
2157 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVCpu, pCpu); break;
2158 default:
2159 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2160 break;
2161 }
2162 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2163}
2164
2165
2166/**
2167 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2168 * TLB entries from the host TLB before VM-entry.
2169 *
2170 * @returns VBox status code.
2171 * @param pVM The cross context VM structure.
2172 */
2173static int hmR0VmxSetupTaggedTlb(PVM pVM)
2174{
2175 /*
2176 * Determine optimal flush type for Nested Paging.
2177 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2178 * guest execution (see hmR3InitFinalizeR0()).
2179 */
2180 if (pVM->hm.s.fNestedPaging)
2181 {
2182 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2183 {
2184 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2185 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2186 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2187 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2188 else
2189 {
2190 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2191 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2192 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2193 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2194 }
2195
2196 /* Make sure the write-back cacheable memory type for EPT is supported. */
2197 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2198 {
2199 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2200 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2201 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2202 }
2203
2204 /* EPT requires a page-walk length of 4. */
2205 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2206 {
2207 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2208 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2209 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2210 }
2211 }
2212 else
2213 {
2214 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2215 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2216 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2217 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2218 }
2219 }
2220
2221 /*
2222 * Determine optimal flush type for VPID.
2223 */
2224 if (pVM->hm.s.vmx.fVpid)
2225 {
2226 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2227 {
2228 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2229 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2230 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2231 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2232 else
2233 {
2234 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2235 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2236 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2237 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2238 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2239 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2240 pVM->hm.s.vmx.fVpid = false;
2241 }
2242 }
2243 else
2244 {
2245 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2246 Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2247 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2248 pVM->hm.s.vmx.fVpid = false;
2249 }
2250 }
2251
2252 /*
2253 * Setup the handler for flushing tagged-TLBs.
2254 */
2255 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2256 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2257 else if (pVM->hm.s.fNestedPaging)
2258 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2259 else if (pVM->hm.s.vmx.fVpid)
2260 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2261 else
2262 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2263 return VINF_SUCCESS;
2264}
2265
2266
2267/**
2268 * Sets up pin-based VM-execution controls in the VMCS.
2269 *
2270 * @returns VBox status code.
2271 * @param pVCpu The cross context virtual CPU structure.
2272 */
2273static int hmR0VmxSetupPinCtls(PVMCPU pVCpu)
2274{
2275 AssertPtr(pVCpu);
2276
2277 PVM pVM = pVCpu->CTX_SUFF(pVM);
2278 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2279 uint32_t fZap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2280
2281 fVal |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2282 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2283
2284 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2285 fVal |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2286
2287 /* Enable the VMX preemption timer. */
2288 if (pVM->hm.s.vmx.fUsePreemptTimer)
2289 {
2290 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2291 fVal |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2292 }
2293
2294#if 0
2295 /* Enable posted-interrupt processing. */
2296 if (pVM->hm.s.fPostedIntrs)
2297 {
2298 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2299 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2300 fVal |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2301 }
2302#endif
2303
2304 if ((fVal & fZap) != fVal)
2305 {
2306 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2307 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, fVal, fZap));
2308 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2309 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2310 }
2311
2312 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2313 AssertRCReturn(rc, rc);
2314
2315 pVCpu->hm.s.vmx.u32PinCtls = fVal;
2316 return rc;
2317}
2318
2319
2320/**
2321 * Sets up processor-based VM-execution controls in the VMCS.
2322 *
2323 * @returns VBox status code.
2324 * @param pVCpu The cross context virtual CPU structure.
2325 */
2326static int hmR0VmxSetupProcCtls(PVMCPU pVCpu)
2327{
2328 AssertPtr(pVCpu);
2329
2330 int rc = VERR_INTERNAL_ERROR_5;
2331 PVM pVM = pVCpu->CTX_SUFF(pVM);
2332 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2333 uint32_t fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2334
2335 fVal |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2336 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2337 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2338 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2339 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2340 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2341 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2342
2343 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2344 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2345 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2346 {
2347 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2348 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2349 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2350 }
2351
2352 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2353 if (!pVM->hm.s.fNestedPaging)
2354 {
2355 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2356 fVal |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2357 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2358 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2359 }
2360
2361 /* Use TPR shadowing if supported by the CPU. */
2362 if ( PDMHasApic(pVM)
2363 && pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2364 {
2365 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2366 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2367 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2368 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2369 AssertRCReturn(rc, rc);
2370
2371 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2372 /* CR8 writes cause a VM-exit based on TPR threshold. */
2373 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2374 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2375 }
2376 else
2377 {
2378 /*
2379 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2380 * Set this control only for 64-bit guests.
2381 */
2382 if (pVM->hm.s.fAllow64BitGuests)
2383 {
2384 fVal |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2385 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2386 }
2387 }
2388
2389 /* Use MSR-bitmaps if supported by the CPU. */
2390 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2391 {
2392 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2393
2394 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2395 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2396 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2397 AssertRCReturn(rc, rc);
2398
2399 /*
2400 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2401 * automatically using dedicated fields in the VMCS.
2402 */
2403 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2404 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2405 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2406 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2407 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2408
2409#if HC_ARCH_BITS == 64
2410 /*
2411 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2412 */
2413 if (pVM->hm.s.fAllow64BitGuests)
2414 {
2415 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2416 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2417 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2418 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2419 }
2420#endif
2421 /*
2422 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2423 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2424 */
2425 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2426 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2427
2428 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2429 }
2430
2431 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2432 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2433 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2434
2435 if ((fVal & fZap) != fVal)
2436 {
2437 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2438 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, fVal, fZap));
2439 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2440 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2441 }
2442
2443 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2444 AssertRCReturn(rc, rc);
2445
2446 pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2447
2448 /*
2449 * Secondary processor-based VM-execution controls.
2450 */
2451 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2452 {
2453 fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2454 fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2455
2456 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2457 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2458
2459 if (pVM->hm.s.fNestedPaging)
2460 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2461
2462 /*
2463 * Enable the INVPCID instruction if supported by the hardware and we expose
2464 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2465 */
2466 if ( (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2467 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2468 {
2469 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2470 }
2471
2472 if (pVM->hm.s.vmx.fVpid)
2473 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2474
2475 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2476 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2477
2478#if 0
2479 if (pVM->hm.s.fVirtApicRegs)
2480 {
2481 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2482 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2483
2484 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2485 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2486 }
2487#endif
2488
2489 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2490 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2491 * done dynamically. */
2492 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2493 {
2494 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2495 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2496 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2497 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2498 AssertRCReturn(rc, rc);
2499 }
2500
2501 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2502 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2503
2504 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2505 && pVM->hm.s.vmx.cPleGapTicks
2506 && pVM->hm.s.vmx.cPleWindowTicks)
2507 {
2508 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2509
2510 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2511 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2512 AssertRCReturn(rc, rc);
2513 }
2514
2515 if ((fVal & fZap) != fVal)
2516 {
2517 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2518 "cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, fVal, fZap));
2519 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2520 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2521 }
2522
2523 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2524 AssertRCReturn(rc, rc);
2525
2526 pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2527 }
2528 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2529 {
2530 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2531 "available\n"));
2532 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2533 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2534 }
2535
2536 return VINF_SUCCESS;
2537}
2538
2539
2540/**
2541 * Sets up miscellaneous (everything other than Pin & Processor-based
2542 * VM-execution) control fields in the VMCS.
2543 *
2544 * @returns VBox status code.
2545 * @param pVCpu The cross context virtual CPU structure.
2546 */
2547static int hmR0VmxSetupMiscCtls(PVMCPU pVCpu)
2548{
2549 AssertPtr(pVCpu);
2550
2551 int rc = VERR_GENERAL_FAILURE;
2552
2553 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2554#if 0
2555 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2556 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2557 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2558
2559 /*
2560 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2561 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2562 * We thus use the exception bitmap to control it rather than use both.
2563 */
2564 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2565 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2566
2567 /* All IO & IOIO instructions cause VM-exits. */
2568 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2569 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2570
2571 /* Initialize the MSR-bitmap area. */
2572 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2573 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2574 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2575 AssertRCReturn(rc, rc);
2576#endif
2577
2578 /* Setup MSR auto-load/store area. */
2579 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2580 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2581 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2582 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2583 AssertRCReturn(rc, rc);
2584
2585 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2586 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2587 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2588 AssertRCReturn(rc, rc);
2589
2590 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2591 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2592 AssertRCReturn(rc, rc);
2593
2594 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2595#if 0
2596 /* Setup debug controls */
2597 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2598 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2599 AssertRCReturn(rc, rc);
2600#endif
2601
2602 return rc;
2603}
2604
2605
2606/**
2607 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2608 *
2609 * We shall setup those exception intercepts that don't change during the
2610 * lifetime of the VM here. The rest are done dynamically while loading the
2611 * guest state.
2612 *
2613 * @returns VBox status code.
2614 * @param pVCpu The cross context virtual CPU structure.
2615 */
2616static int hmR0VmxInitXcptBitmap(PVMCPU pVCpu)
2617{
2618 AssertPtr(pVCpu);
2619
2620 uint32_t u32XcptBitmap;
2621
2622 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2623 u32XcptBitmap = RT_BIT_32(X86_XCPT_AC);
2624
2625 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2626 and writes, and because recursive #DBs can cause the CPU hang, we must always
2627 intercept #DB. */
2628 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2629
2630 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2631 if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
2632 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2633
2634 /* Commit it to the VMCS. */
2635 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2636 AssertRCReturn(rc, rc);
2637
2638 /* Update our cache of the exception bitmap. */
2639 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2640 return VINF_SUCCESS;
2641}
2642
2643
2644/**
2645 * Does per-VM VT-x initialization.
2646 *
2647 * @returns VBox status code.
2648 * @param pVM The cross context VM structure.
2649 */
2650VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2651{
2652 LogFlowFunc(("pVM=%p\n", pVM));
2653
2654 int rc = hmR0VmxStructsAlloc(pVM);
2655 if (RT_FAILURE(rc))
2656 {
2657 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2658 return rc;
2659 }
2660
2661 return VINF_SUCCESS;
2662}
2663
2664
2665/**
2666 * Does per-VM VT-x termination.
2667 *
2668 * @returns VBox status code.
2669 * @param pVM The cross context VM structure.
2670 */
2671VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2672{
2673 LogFlowFunc(("pVM=%p\n", pVM));
2674
2675#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2676 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2677 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2678#endif
2679 hmR0VmxStructsFree(pVM);
2680 return VINF_SUCCESS;
2681}
2682
2683
2684/**
2685 * Sets up the VM for execution under VT-x.
2686 * This function is only called once per-VM during initialization.
2687 *
2688 * @returns VBox status code.
2689 * @param pVM The cross context VM structure.
2690 */
2691VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2692{
2693 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2694 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2695
2696 LogFlowFunc(("pVM=%p\n", pVM));
2697
2698 /*
2699 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be
2700 * allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2701 * pRealModeTSS, see hmR3InitFinalizeR0Intel().
2702 */
2703 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2704 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2705 || !pVM->hm.s.vmx.pRealModeTSS))
2706 {
2707 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2708 return VERR_INTERNAL_ERROR;
2709 }
2710
2711 /* Initialize these always, see hmR3InitFinalizeR0().*/
2712 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2713 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2714
2715 /* Setup the tagged-TLB flush handlers. */
2716 int rc = hmR0VmxSetupTaggedTlb(pVM);
2717 if (RT_FAILURE(rc))
2718 {
2719 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2720 return rc;
2721 }
2722
2723 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2724 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2725#if HC_ARCH_BITS == 64
2726 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2727 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2728 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2729 {
2730 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2731 }
2732#endif
2733
2734 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2735 RTCCUINTREG uHostCR4 = ASMGetCR4();
2736 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2737 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2738
2739 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2740 {
2741 PVMCPU pVCpu = &pVM->aCpus[i];
2742 AssertPtr(pVCpu);
2743 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2744
2745 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2746 Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2747
2748 /* Set revision dword at the beginning of the VMCS structure. */
2749 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2750
2751 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2752 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2753 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc\n", rc),
2754 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2755
2756 /* Load this VMCS as the current VMCS. */
2757 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2758 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc\n", rc),
2759 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2760
2761 rc = hmR0VmxSetupPinCtls(pVCpu);
2762 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc\n", rc),
2763 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2764
2765 rc = hmR0VmxSetupProcCtls(pVCpu);
2766 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc\n", rc),
2767 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2768
2769 rc = hmR0VmxSetupMiscCtls(pVCpu);
2770 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc\n", rc),
2771 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2772
2773 rc = hmR0VmxInitXcptBitmap(pVCpu);
2774 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc\n", rc),
2775 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2776
2777#if HC_ARCH_BITS == 32
2778 rc = hmR0VmxInitVmcsReadCache(pVCpu);
2779 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc\n", rc),
2780 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2781#endif
2782
2783 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2784 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2785 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc\n", rc),
2786 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2787
2788 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2789
2790 hmR0VmxUpdateErrorRecord(pVCpu, rc);
2791 }
2792
2793 return VINF_SUCCESS;
2794}
2795
2796
2797/**
2798 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2799 * the VMCS.
2800 *
2801 * @returns VBox status code.
2802 */
2803static int hmR0VmxExportHostControlRegs(void)
2804{
2805 RTCCUINTREG uReg = ASMGetCR0();
2806 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2807 AssertRCReturn(rc, rc);
2808
2809 uReg = ASMGetCR3();
2810 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2811 AssertRCReturn(rc, rc);
2812
2813 uReg = ASMGetCR4();
2814 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2815 AssertRCReturn(rc, rc);
2816 return rc;
2817}
2818
2819
2820/**
2821 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2822 * the host-state area in the VMCS.
2823 *
2824 * @returns VBox status code.
2825 * @param pVCpu The cross context virtual CPU structure.
2826 */
2827static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2828{
2829#if HC_ARCH_BITS == 64
2830/**
2831 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2832 * requirements. See hmR0VmxExportHostSegmentRegs().
2833 */
2834# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2835 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2836 { \
2837 bool fValidSelector = true; \
2838 if ((selValue) & X86_SEL_LDT) \
2839 { \
2840 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2841 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2842 } \
2843 if (fValidSelector) \
2844 { \
2845 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2846 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2847 } \
2848 (selValue) = 0; \
2849 }
2850
2851 /*
2852 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2853 * should -not- save the messed up state without restoring the original host-state,
2854 * see @bugref{7240}.
2855 *
2856 * This apparently can happen (most likely the FPU changes), deal with it rather than
2857 * asserting. Was observed booting Solaris 10u10 32-bit guest.
2858 */
2859 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2860 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2861 {
2862 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2863 pVCpu->idCpu));
2864 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2865 }
2866 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2867#else
2868 RT_NOREF(pVCpu);
2869#endif
2870
2871 /*
2872 * Host DS, ES, FS and GS segment registers.
2873 */
2874#if HC_ARCH_BITS == 64
2875 RTSEL uSelDS = ASMGetDS();
2876 RTSEL uSelES = ASMGetES();
2877 RTSEL uSelFS = ASMGetFS();
2878 RTSEL uSelGS = ASMGetGS();
2879#else
2880 RTSEL uSelDS = 0;
2881 RTSEL uSelES = 0;
2882 RTSEL uSelFS = 0;
2883 RTSEL uSelGS = 0;
2884#endif
2885
2886 /*
2887 * Host CS and SS segment registers.
2888 */
2889 RTSEL uSelCS = ASMGetCS();
2890 RTSEL uSelSS = ASMGetSS();
2891
2892 /*
2893 * Host TR segment register.
2894 */
2895 RTSEL uSelTR = ASMGetTR();
2896
2897#if HC_ARCH_BITS == 64
2898 /*
2899 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2900 * gain VM-entry and restore them before we get preempted.
2901 *
2902 * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2903 */
2904 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2905 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2906 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2907 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2908# undef VMXLOCAL_ADJUST_HOST_SEG
2909#endif
2910
2911 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2912 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2913 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2914 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2915 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2916 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2917 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2918 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2919 Assert(uSelCS);
2920 Assert(uSelTR);
2921
2922 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2923#if 0
2924 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2925 Assert(uSelSS != 0);
2926#endif
2927
2928 /* Write these host selector fields into the host-state area in the VMCS. */
2929 int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2930 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2931#if HC_ARCH_BITS == 64
2932 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2933 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2934 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2935 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2936#else
2937 NOREF(uSelDS);
2938 NOREF(uSelES);
2939 NOREF(uSelFS);
2940 NOREF(uSelGS);
2941#endif
2942 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2943 AssertRCReturn(rc, rc);
2944
2945 /*
2946 * Host GDTR and IDTR.
2947 */
2948 RTGDTR Gdtr;
2949 RTIDTR Idtr;
2950 RT_ZERO(Gdtr);
2951 RT_ZERO(Idtr);
2952 ASMGetGDTR(&Gdtr);
2953 ASMGetIDTR(&Idtr);
2954 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2955 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2956 AssertRCReturn(rc, rc);
2957
2958#if HC_ARCH_BITS == 64
2959 /*
2960 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
2961 * them to the maximum limit (0xffff) on every VM-exit.
2962 */
2963 if (Gdtr.cbGdt != 0xffff)
2964 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2965
2966 /*
2967 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2968 * Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
2969 * as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
2970 * However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
2971 * uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
2972 * alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
2973 * at 0xffff on hosts where we are sure it won't cause trouble.
2974 */
2975# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
2976 if (Idtr.cbIdt < 0x0fff)
2977# else
2978 if (Idtr.cbIdt != 0xffff)
2979# endif
2980 {
2981 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
2982 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2983 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2984 }
2985#endif
2986
2987 /*
2988 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
2989 * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
2990 * RPL should be too in most cases.
2991 */
2992 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
2993 ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
2994
2995 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2996#if HC_ARCH_BITS == 64
2997 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
2998
2999 /*
3000 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3001 * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3002 * restoration if the host has something else. Task switching is not supported in 64-bit
3003 * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3004 * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3005 *
3006 * [1] See Intel spec. 3.5 "System Descriptor Types".
3007 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3008 */
3009 PVM pVM = pVCpu->CTX_SUFF(pVM);
3010 Assert(pDesc->System.u4Type == 11);
3011 if ( pDesc->System.u16LimitLow != 0x67
3012 || pDesc->System.u4LimitHigh)
3013 {
3014 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3015 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3016 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3017 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3018 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3019 }
3020
3021 /*
3022 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3023 */
3024 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3025 {
3026 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3027 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3028 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3029 {
3030 /* The GDT is read-only but the writable GDT is available. */
3031 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3032 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3033 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3034 AssertRCReturn(rc, rc);
3035 }
3036 }
3037#else
3038 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3039#endif
3040 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3041 AssertRCReturn(rc, rc);
3042
3043 /*
3044 * Host FS base and GS base.
3045 */
3046#if HC_ARCH_BITS == 64
3047 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3048 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3049 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3050 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3051 AssertRCReturn(rc, rc);
3052
3053 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3054 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3055 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3056 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3057 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3058#endif
3059 return VINF_SUCCESS;
3060}
3061
3062
3063/**
3064 * Exports certain host MSRs in the VM-exit MSR-load area and some in the
3065 * host-state area of the VMCS.
3066 *
3067 * Theses MSRs will be automatically restored on the host after every successful
3068 * VM-exit.
3069 *
3070 * @returns VBox status code.
3071 * @param pVCpu The cross context virtual CPU structure.
3072 *
3073 * @remarks No-long-jump zone!!!
3074 */
3075static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3076{
3077 AssertPtr(pVCpu);
3078 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3079
3080 /*
3081 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3082 * rather than swapping them on every VM-entry.
3083 */
3084 hmR0VmxLazySaveHostMsrs(pVCpu);
3085
3086 /*
3087 * Host Sysenter MSRs.
3088 */
3089 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3090#if HC_ARCH_BITS == 32
3091 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3092 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3093#else
3094 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3095 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3096#endif
3097 AssertRCReturn(rc, rc);
3098
3099 /*
3100 * Host EFER MSR.
3101 *
3102 * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3103 * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3104 */
3105 PVM pVM = pVCpu->CTX_SUFF(pVM);
3106 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3107 {
3108 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3109 AssertRCReturn(rc, rc);
3110 }
3111
3112 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3113
3114 return VINF_SUCCESS;
3115}
3116
3117
3118/**
3119 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3120 *
3121 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3122 * these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3123 * hmR0VMxExportGuestEntryCtls().
3124 *
3125 * @returns true if we need to load guest EFER, false otherwise.
3126 * @param pVCpu The cross context virtual CPU structure.
3127 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3128 * out-of-sync. Make sure to update the required fields
3129 * before using them.
3130 *
3131 * @remarks Requires EFER, CR4.
3132 * @remarks No-long-jump zone!!!
3133 */
3134static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3135{
3136#ifdef HMVMX_ALWAYS_SWAP_EFER
3137 return true;
3138#endif
3139
3140#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3141 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3142 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3143 return false;
3144#endif
3145
3146 PVM pVM = pVCpu->CTX_SUFF(pVM);
3147 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3148 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3149
3150 /*
3151 * For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3152 * guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3153 */
3154 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3155 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3156 {
3157 return true;
3158 }
3159
3160 /*
3161 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3162 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3163 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3164 */
3165 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3166 && (pMixedCtx->cr0 & X86_CR0_PG)
3167 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3168 {
3169 /* Assert that host is PAE capable. */
3170 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3171 return true;
3172 }
3173
3174 return false;
3175}
3176
3177
3178/**
3179 * Exports the guest state with appropriate VM-entry controls in the VMCS.
3180 *
3181 * These controls can affect things done on VM-exit; e.g. "load debug controls",
3182 * see Intel spec. 24.8.1 "VM-entry controls".
3183 *
3184 * @returns VBox status code.
3185 * @param pVCpu The cross context virtual CPU structure.
3186 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3187 * out-of-sync. Make sure to update the required fields
3188 * before using them.
3189 *
3190 * @remarks Requires EFER.
3191 * @remarks No-long-jump zone!!!
3192 */
3193static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3194{
3195 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3196 {
3197 PVM pVM = pVCpu->CTX_SUFF(pVM);
3198 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3199 uint32_t fZap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3200
3201 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3202 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3203
3204 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3205 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3206 {
3207 fVal |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3208 Log4Func(("VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n"));
3209 }
3210 else
3211 Assert(!(fVal & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3212
3213 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3214 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3215 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3216 {
3217 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3218 Log4Func(("VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n"));
3219 }
3220
3221 /*
3222 * The following should -not- be set (since we're not in SMM mode):
3223 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3224 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3225 */
3226
3227 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3228 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3229
3230 if ((fVal & fZap) != fVal)
3231 {
3232 Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3233 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, fVal, fZap));
3234 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3235 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3236 }
3237
3238 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3239 AssertRCReturn(rc, rc);
3240
3241 pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3242 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3243 }
3244 return VINF_SUCCESS;
3245}
3246
3247
3248/**
3249 * Exports the guest state with appropriate VM-exit controls in the VMCS.
3250 *
3251 * @returns VBox status code.
3252 * @param pVCpu The cross context virtual CPU structure.
3253 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3254 * out-of-sync. Make sure to update the required fields
3255 * before using them.
3256 *
3257 * @remarks Requires EFER.
3258 */
3259static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3260{
3261 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3262 {
3263 PVM pVM = pVCpu->CTX_SUFF(pVM);
3264 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3265 uint32_t fZap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3266
3267 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3268 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3269
3270 /*
3271 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3272 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3273 * hmR0VmxExportHostMsrs().
3274 */
3275#if HC_ARCH_BITS == 64
3276 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3277 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3278#else
3279 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3280 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3281 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3282 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3283 {
3284 /* The switcher returns to long mode, EFER is managed by the switcher. */
3285 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3286 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3287 }
3288 else
3289 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3290#endif
3291
3292 /* If the newer VMCS fields for managing EFER exists, use it. */
3293 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3294 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3295 {
3296 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3297 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3298 Log4Func(("VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR and VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n"));
3299 }
3300
3301 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3302 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3303
3304 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3305 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3306 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3307
3308 if ( pVM->hm.s.vmx.fUsePreemptTimer
3309 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3310 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3311
3312 if ((fVal & fZap) != fVal)
3313 {
3314 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3315 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, fVal, fZap));
3316 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3317 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3318 }
3319
3320 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3321 AssertRCReturn(rc, rc);
3322
3323 pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3324 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3325 }
3326 return VINF_SUCCESS;
3327}
3328
3329
3330/**
3331 * Sets the TPR threshold in the VMCS.
3332 *
3333 * @returns VBox status code.
3334 * @param pVCpu The cross context virtual CPU structure.
3335 * @param u32TprThreshold The TPR threshold (task-priority class only).
3336 */
3337DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3338{
3339 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3340 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3341 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3342}
3343
3344
3345/**
3346 * Exports the guest APIC TPR state into the VMCS.
3347 *
3348 * @returns VBox status code.
3349 * @param pVCpu The cross context virtual CPU structure.
3350 *
3351 * @remarks No-long-jump zone!!!
3352 */
3353static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3354{
3355 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3356 {
3357 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3358 && APICIsEnabled(pVCpu))
3359 {
3360 /*
3361 * Setup TPR shadowing.
3362 */
3363 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3364 {
3365 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3366
3367 bool fPendingIntr = false;
3368 uint8_t u8Tpr = 0;
3369 uint8_t u8PendingIntr = 0;
3370 int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3371 AssertRCReturn(rc, rc);
3372
3373 /*
3374 * If there are interrupts pending but masked by the TPR, instruct VT-x to
3375 * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3376 * priority of the pending interrupt so we can deliver the interrupt. If there
3377 * are no interrupts pending, set threshold to 0 to not cause any
3378 * TPR-below-threshold VM-exits.
3379 */
3380 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3381 uint32_t u32TprThreshold = 0;
3382 if (fPendingIntr)
3383 {
3384 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3385 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3386 const uint8_t u8TprPriority = u8Tpr >> 4;
3387 if (u8PendingPriority <= u8TprPriority)
3388 u32TprThreshold = u8PendingPriority;
3389 }
3390
3391 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3392 AssertRCReturn(rc, rc);
3393 }
3394 }
3395 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3396 }
3397 return VINF_SUCCESS;
3398}
3399
3400
3401/**
3402 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3403 *
3404 * @returns Guest's interruptibility-state.
3405 * @param pVCpu The cross context virtual CPU structure.
3406 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3407 * out-of-sync. Make sure to update the required fields
3408 * before using them.
3409 *
3410 * @remarks No-long-jump zone!!!
3411 */
3412static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3413{
3414 /*
3415 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3416 */
3417 uint32_t fIntrState = 0;
3418 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3419 {
3420 /* If inhibition is active, RIP & RFLAGS should've been accessed
3421 (i.e. read previously from the VMCS or from ring-3). */
3422#ifdef VBOX_STRICT
3423 uint64_t const fExtrn = ASMAtomicUoReadU64(&pMixedCtx->fExtrn);
3424 AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3425#endif
3426 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3427 {
3428 if (pMixedCtx->eflags.Bits.u1IF)
3429 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3430 else
3431 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3432 }
3433 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3434 {
3435 /*
3436 * We can clear the inhibit force flag as even if we go back to the recompiler
3437 * without executing guest code in VT-x, the flag's condition to be cleared is
3438 * met and thus the cleared state is correct.
3439 */
3440 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3441 }
3442 }
3443
3444 /*
3445 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3446 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3447 * setting this would block host-NMIs and IRET will not clear the blocking.
3448 *
3449 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3450 */
3451 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3452 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3453 {
3454 fIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3455 }
3456
3457 return fIntrState;
3458}
3459
3460
3461/**
3462 * Exports the guest's interruptibility-state into the guest-state area in the
3463 * VMCS.
3464 *
3465 * @returns VBox status code.
3466 * @param pVCpu The cross context virtual CPU structure.
3467 * @param fIntrState The interruptibility-state to set.
3468 */
3469static int hmR0VmxExportGuestIntrState(PVMCPU pVCpu, uint32_t fIntrState)
3470{
3471 NOREF(pVCpu);
3472 AssertMsg(!(fIntrState & 0xfffffff0), ("%#x\n", fIntrState)); /* Bits 31:4 MBZ. */
3473 Assert((fIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3474 return VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, fIntrState);
3475}
3476
3477
3478/**
3479 * Exports the exception intercepts required for guest execution in the VMCS.
3480 *
3481 * @returns VBox status code.
3482 * @param pVCpu The cross context virtual CPU structure.
3483 *
3484 * @remarks No-long-jump zone!!!
3485 */
3486static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3487{
3488 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3489 {
3490 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3491 if (pVCpu->hm.s.fGIMTrapXcptUD)
3492 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3493#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3494 else
3495 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3496#endif
3497
3498 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3499 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3500
3501 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3502 AssertRCReturn(rc, rc);
3503
3504 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3505 Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", pVCpu->hm.s.vmx.u32XcptBitmap));
3506 }
3507 return VINF_SUCCESS;
3508}
3509
3510
3511/**
3512 * Exports the guest's RIP into the guest-state area in the VMCS.
3513 *
3514 * @returns VBox status code.
3515 * @param pVCpu The cross context virtual CPU structure.
3516 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3517 * out-of-sync. Make sure to update the required fields
3518 * before using them.
3519 *
3520 * @remarks No-long-jump zone!!!
3521 */
3522static int hmR0VmxExportGuestRip(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3523{
3524 int rc = VINF_SUCCESS;
3525 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3526 {
3527 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3528 AssertRCReturn(rc, rc);
3529
3530 /* Update the exit history entry with the correct CS.BASE + RIP or just RIP. */
3531 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
3532 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
3533 else
3534 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->rip, false);
3535
3536 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3537 Log4Func(("RIP=%#RX64\n", pMixedCtx->rip));
3538 }
3539 return rc;
3540}
3541
3542
3543/**
3544 * Exports the guest's RSP into the guest-state area in the VMCS.
3545 *
3546 * @returns VBox status code.
3547 * @param pVCpu The cross context virtual CPU structure.
3548 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3549 * out-of-sync. Make sure to update the required fields
3550 * before using them.
3551 *
3552 * @remarks No-long-jump zone!!!
3553 */
3554static int hmR0VmxExportGuestRsp(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3555{
3556 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3557 {
3558 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3559 AssertRCReturn(rc, rc);
3560
3561 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3562 }
3563 return VINF_SUCCESS;
3564}
3565
3566
3567/**
3568 * Exports the guest's RFLAGS into the guest-state area in the VMCS.
3569 *
3570 * @returns VBox status code.
3571 * @param pVCpu The cross context virtual CPU structure.
3572 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3573 * out-of-sync. Make sure to update the required fields
3574 * before using them.
3575 *
3576 * @remarks No-long-jump zone!!!
3577 */
3578static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3579{
3580 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3581 {
3582 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3583 Let us assert it as such and use 32-bit VMWRITE. */
3584 Assert(!RT_HI_U32(pMixedCtx->rflags.u64));
3585 X86EFLAGS fEFlags = pMixedCtx->eflags;
3586 Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3587 Assert(!(fEFlags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3588
3589 /*
3590 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3591 * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3592 * can run the real-mode guest code under Virtual 8086 mode.
3593 */
3594 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3595 {
3596 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3597 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3598 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3599 fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3600 fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3601 }
3602
3603 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3604 AssertRCReturn(rc, rc);
3605
3606 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3607 Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3608 }
3609 return VINF_SUCCESS;
3610}
3611
3612
3613/**
3614 * Exports the guest CR0 control register into the guest-state area in the VMCS.
3615 *
3616 * The guest FPU state is always pre-loaded hence we don't need to bother about
3617 * sharing FPU related CR0 bits between the guest and host.
3618 *
3619 * @returns VBox status code.
3620 * @param pVCpu The cross context virtual CPU structure.
3621 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3622 * out-of-sync. Make sure to update the required fields
3623 * before using them.
3624 *
3625 * @remarks No-long-jump zone!!!
3626 */
3627static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3628{
3629 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3630 {
3631 PVM pVM = pVCpu->CTX_SUFF(pVM);
3632 Assert(!RT_HI_U32(pMixedCtx->cr0));
3633 uint32_t const uShadowCR0 = pMixedCtx->cr0;
3634 uint32_t uGuestCR0 = pMixedCtx->cr0;
3635
3636 /*
3637 * Setup VT-x's view of the guest CR0.
3638 * Minimize VM-exits due to CR3 changes when we have NestedPaging.
3639 */
3640 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3641 if (pVM->hm.s.fNestedPaging)
3642 {
3643 if (CPUMIsGuestPagingEnabled(pVCpu))
3644 {
3645 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3646 uProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3647 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3648 }
3649 else
3650 {
3651 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3652 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3653 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3654 }
3655
3656 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3657 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3658 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3659 }
3660 else
3661 {
3662 /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3663 uGuestCR0 |= X86_CR0_WP;
3664 }
3665
3666 /*
3667 * Guest FPU bits.
3668 *
3669 * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3670 * using CR0.TS.
3671 *
3672 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3673 * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3674 */
3675 uGuestCR0 |= X86_CR0_NE;
3676
3677 /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3678 bool const fInterceptMF = !(uShadowCR0 & X86_CR0_NE);
3679
3680 /*
3681 * Update exception intercepts.
3682 */
3683 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3684 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3685 {
3686 Assert(PDMVmmDevHeapIsEnabled(pVM));
3687 Assert(pVM->hm.s.vmx.pRealModeTSS);
3688 uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3689 }
3690 else
3691 {
3692 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3693 uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3694 if (fInterceptMF)
3695 uXcptBitmap |= RT_BIT(X86_XCPT_MF);
3696 }
3697
3698 /* Additional intercepts for debugging, define these yourself explicitly. */
3699#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3700 uXcptBitmap |= 0
3701 | RT_BIT(X86_XCPT_BP)
3702 | RT_BIT(X86_XCPT_DE)
3703 | RT_BIT(X86_XCPT_NM)
3704 | RT_BIT(X86_XCPT_TS)
3705 | RT_BIT(X86_XCPT_UD)
3706 | RT_BIT(X86_XCPT_NP)
3707 | RT_BIT(X86_XCPT_SS)
3708 | RT_BIT(X86_XCPT_GP)
3709 | RT_BIT(X86_XCPT_PF)
3710 | RT_BIT(X86_XCPT_MF)
3711 ;
3712#elif defined(HMVMX_ALWAYS_TRAP_PF)
3713 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
3714#endif
3715 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3716 {
3717 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3718 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3719 }
3720 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3721
3722 /*
3723 * Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3724 */
3725 uint32_t fSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3726 uint32_t fZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3727 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3728 fSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3729 else
3730 Assert((fSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3731
3732 uGuestCR0 |= fSetCR0;
3733 uGuestCR0 &= fZapCR0;
3734 uGuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3735
3736 /*
3737 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3738 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3739 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3740 */
3741 uint32_t uCR0Mask = X86_CR0_PE
3742 | X86_CR0_NE
3743 | (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3744 | X86_CR0_PG
3745 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3746 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3747 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3748
3749 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3750 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3751 * and @bugref{6944}. */
3752#if 0
3753 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3754 uCr0Mask &= ~X86_CR0_PE;
3755#endif
3756 /*
3757 * Finally, update VMCS fields with the CR0 values.
3758 */
3759 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, uGuestCR0);
3760 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, uShadowCR0);
3761 if (uCR0Mask != pVCpu->hm.s.vmx.u32CR0Mask)
3762 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, uCR0Mask);
3763 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3764 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3765 AssertRCReturn(rc, rc);
3766
3767 pVCpu->hm.s.vmx.u32CR0Mask = uCR0Mask;
3768 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3769
3770 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3771
3772 Log4Func(("uCr0Mask=%#RX32 uShadowCR0=%#RX32 uGuestCR0=%#RX32 (fSetCR0=%#RX32 fZapCR0=%#RX32\n", uCR0Mask, uShadowCR0,
3773 uGuestCR0, fSetCR0, fZapCR0));
3774 }
3775
3776 return VINF_SUCCESS;
3777}
3778
3779
3780/**
3781 * Exports the guest control registers (CR3, CR4) into the guest-state area
3782 * in the VMCS.
3783 *
3784 * @returns VBox strict status code.
3785 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3786 * without unrestricted guest access and the VMMDev is not presently
3787 * mapped (e.g. EFI32).
3788 *
3789 * @param pVCpu The cross context virtual CPU structure.
3790 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3791 * out-of-sync. Make sure to update the required fields
3792 * before using them.
3793 *
3794 * @remarks No-long-jump zone!!!
3795 */
3796static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3797{
3798 int rc = VINF_SUCCESS;
3799 PVM pVM = pVCpu->CTX_SUFF(pVM);
3800
3801 /*
3802 * Guest CR2.
3803 * It's always loaded in the assembler code. Nothing to do here.
3804 */
3805
3806 /*
3807 * Guest CR3.
3808 */
3809 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3810 {
3811 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3812 if (pVM->hm.s.fNestedPaging)
3813 {
3814 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3815
3816 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3817 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3818 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3819 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3820
3821 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3822 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3823 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3824
3825 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3826 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3827 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3828 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3829 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3830 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3831 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3832
3833 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3834 AssertRCReturn(rc, rc);
3835
3836 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3837 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3838 {
3839 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3840 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3841 {
3842 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3843 AssertRCReturn(rc, rc);
3844 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3845 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3846 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3847 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3848 AssertRCReturn(rc, rc);
3849 }
3850
3851 /*
3852 * The guest's view of its CR3 is unblemished with Nested Paging when the
3853 * guest is using paging or we have unrestricted guest execution to handle
3854 * the guest when it's not using paging.
3855 */
3856 GCPhysGuestCR3 = pMixedCtx->cr3;
3857 }
3858 else
3859 {
3860 /*
3861 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
3862 * thinks it accesses physical memory directly, we use our identity-mapped
3863 * page table to map guest-linear to guest-physical addresses. EPT takes care
3864 * of translating it to host-physical addresses.
3865 */
3866 RTGCPHYS GCPhys;
3867 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3868
3869 /* We obtain it here every time as the guest could have relocated this PCI region. */
3870 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3871 if (RT_SUCCESS(rc))
3872 { /* likely */ }
3873 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3874 {
3875 Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3876 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3877 }
3878 else
3879 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3880
3881 GCPhysGuestCR3 = GCPhys;
3882 }
3883
3884 Log4Func(("uGuestCR3=%#RGp (GstN)\n", GCPhysGuestCR3));
3885 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3886 AssertRCReturn(rc, rc);
3887 }
3888 else
3889 {
3890 /* Non-nested paging case, just use the hypervisor's CR3. */
3891 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3892
3893 Log4Func(("uGuestCR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3894 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3895 AssertRCReturn(rc, rc);
3896 }
3897
3898 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3899 }
3900
3901 /*
3902 * Guest CR4.
3903 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3904 */
3905 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3906 {
3907 Assert(!RT_HI_U32(pMixedCtx->cr4));
3908 uint32_t uGuestCR4 = pMixedCtx->cr4;
3909 uint32_t const uShadowCR4 = pMixedCtx->cr4;
3910
3911 /*
3912 * Setup VT-x's view of the guest CR4.
3913 *
3914 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3915 * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3916 * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3917 *
3918 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3919 */
3920 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3921 {
3922 Assert(pVM->hm.s.vmx.pRealModeTSS);
3923 Assert(PDMVmmDevHeapIsEnabled(pVM));
3924 uGuestCR4 &= ~X86_CR4_VME;
3925 }
3926
3927 if (pVM->hm.s.fNestedPaging)
3928 {
3929 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3930 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3931 {
3932 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3933 uGuestCR4 |= X86_CR4_PSE;
3934 /* Our identity mapping is a 32-bit page directory. */
3935 uGuestCR4 &= ~X86_CR4_PAE;
3936 }
3937 /* else use guest CR4.*/
3938 }
3939 else
3940 {
3941 /*
3942 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3943 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3944 */
3945 switch (pVCpu->hm.s.enmShadowMode)
3946 {
3947 case PGMMODE_REAL: /* Real-mode. */
3948 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3949 case PGMMODE_32_BIT: /* 32-bit paging. */
3950 {
3951 uGuestCR4 &= ~X86_CR4_PAE;
3952 break;
3953 }
3954
3955 case PGMMODE_PAE: /* PAE paging. */
3956 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3957 {
3958 uGuestCR4 |= X86_CR4_PAE;
3959 break;
3960 }
3961
3962 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3963 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3964#ifdef VBOX_ENABLE_64_BITS_GUESTS
3965 break;
3966#endif
3967 default:
3968 AssertFailed();
3969 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3970 }
3971 }
3972
3973 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3974 uint64_t fSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3975 uint64_t fZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3976 uGuestCR4 |= fSetCR4;
3977 uGuestCR4 &= fZapCR4;
3978
3979 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
3980 uint32_t u32CR4Mask = X86_CR4_VME
3981 | X86_CR4_PAE
3982 | X86_CR4_PGE
3983 | X86_CR4_PSE
3984 | X86_CR4_VMXE;
3985 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
3986 u32CR4Mask |= X86_CR4_OSXSAVE;
3987 if (pVM->cpum.ro.GuestFeatures.fPcid)
3988 u32CR4Mask |= X86_CR4_PCIDE;
3989
3990 /* Write VT-x's view of the guest CR4, the CR4 modify mask and the read-only CR4 shadow into the VMCS. */
3991 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, uGuestCR4);
3992 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, uShadowCR4);
3993 if (pVCpu->hm.s.vmx.u32CR4Mask != u32CR4Mask)
3994 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3995 AssertRCReturn(rc, rc);
3996 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3997
3998 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
3999 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4000
4001 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4002
4003 Log4Func(("uGuestCR4=%#RX32 uShadowCR4=%#RX32 (fSetCR4=%#RX32 fZapCR4=%#RX32)\n", uGuestCR4, uShadowCR4, fSetCR4,
4004 fZapCR4));
4005 }
4006 return rc;
4007}
4008
4009
4010/**
4011 * Exports the guest debug registers into the guest-state area in the VMCS.
4012 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4013 *
4014 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4015 *
4016 * @returns VBox status code.
4017 * @param pVCpu The cross context virtual CPU structure.
4018 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4019 * out-of-sync. Make sure to update the required fields
4020 * before using them.
4021 *
4022 * @remarks No-long-jump zone!!!
4023 */
4024static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4025{
4026 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4027
4028#ifdef VBOX_STRICT
4029 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4030 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4031 {
4032 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4033 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4034 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4035 }
4036#endif
4037
4038 bool fSteppingDB = false;
4039 bool fInterceptMovDRx = false;
4040 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4041 if (pVCpu->hm.s.fSingleInstruction)
4042 {
4043 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4044 PVM pVM = pVCpu->CTX_SUFF(pVM);
4045 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4046 {
4047 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4048 Assert(fSteppingDB == false);
4049 }
4050 else
4051 {
4052 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4053 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
4054 pVCpu->hm.s.fClearTrapFlag = true;
4055 fSteppingDB = true;
4056 }
4057 }
4058
4059 uint32_t uGuestDR7;
4060 if ( fSteppingDB
4061 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4062 {
4063 /*
4064 * Use the combined guest and host DRx values found in the hypervisor register set
4065 * because the debugger has breakpoints active or someone is single stepping on the
4066 * host side without a monitor trap flag.
4067 *
4068 * Note! DBGF expects a clean DR6 state before executing guest code.
4069 */
4070#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4071 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4072 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4073 {
4074 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4075 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4076 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4077 }
4078 else
4079#endif
4080 if (!CPUMIsHyperDebugStateActive(pVCpu))
4081 {
4082 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4083 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4084 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4085 }
4086
4087 /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4088 uGuestDR7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4089 pVCpu->hm.s.fUsingHyperDR7 = true;
4090 fInterceptMovDRx = true;
4091 }
4092 else
4093 {
4094 /*
4095 * If the guest has enabled debug registers, we need to load them prior to
4096 * executing guest code so they'll trigger at the right time.
4097 */
4098 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD))
4099 {
4100#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4101 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4102 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4103 {
4104 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4105 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4106 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4107 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4108 }
4109 else
4110#endif
4111 if (!CPUMIsGuestDebugStateActive(pVCpu))
4112 {
4113 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4114 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4115 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4116 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4117 }
4118 Assert(!fInterceptMovDRx);
4119 }
4120 /*
4121 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4122 * must intercept #DB in order to maintain a correct DR6 guest value, and
4123 * because we need to intercept it to prevent nested #DBs from hanging the
4124 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4125 */
4126#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4127 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4128 && !CPUMIsGuestDebugStateActive(pVCpu))
4129#else
4130 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4131#endif
4132 {
4133 fInterceptMovDRx = true;
4134 }
4135
4136 /* Update DR7 with the actual guest value. */
4137 uGuestDR7 = pMixedCtx->dr[7];
4138 pVCpu->hm.s.fUsingHyperDR7 = false;
4139 }
4140
4141 if (fInterceptMovDRx)
4142 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4143 else
4144 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4145
4146 /*
4147 * Update the processor-based VM-execution controls for MOV-DRx intercepts and the monitor-trap flag.
4148 */
4149 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4150 {
4151 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4152 AssertRCReturn(rc2, rc2);
4153 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4154 }
4155
4156 /*
4157 * Update guest DR7.
4158 */
4159 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, uGuestDR7);
4160 AssertRCReturn(rc, rc);
4161
4162 return VINF_SUCCESS;
4163}
4164
4165
4166#ifdef VBOX_STRICT
4167/**
4168 * Strict function to validate segment registers.
4169 *
4170 * @param pVCpu The cross context virtual CPU structure.
4171 * @param pCtx Pointer to the guest-CPU context.
4172 *
4173 * @remarks Will import guest CR0 on strict builds during validation of
4174 * segments.
4175 */
4176static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PCCPUMCTX pCtx)
4177{
4178 /*
4179 * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4180 *
4181 * The reason we check for attribute value 0 in this function and not just the unusable bit is
4182 * because hmR0VmxWriteSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4183 * and doesn't change the guest-context value.
4184 */
4185 PVM pVM = pVCpu->CTX_SUFF(pVM);
4186 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4187 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4188 && ( !CPUMIsGuestInRealModeEx(pCtx)
4189 && !CPUMIsGuestInV86ModeEx(pCtx)))
4190 {
4191 /* Protected mode checks */
4192 /* CS */
4193 Assert(pCtx->cs.Attr.n.u1Present);
4194 Assert(!(pCtx->cs.Attr.u & 0xf00));
4195 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4196 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4197 || !(pCtx->cs.Attr.n.u1Granularity));
4198 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4199 || (pCtx->cs.Attr.n.u1Granularity));
4200 /* CS cannot be loaded with NULL in protected mode. */
4201 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4202 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4203 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4204 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4205 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4206 else
4207 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4208 /* SS */
4209 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4210 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4211 if ( !(pCtx->cr0 & X86_CR0_PE)
4212 || pCtx->cs.Attr.n.u4Type == 3)
4213 {
4214 Assert(!pCtx->ss.Attr.n.u2Dpl);
4215 }
4216 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4217 {
4218 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4219 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4220 Assert(pCtx->ss.Attr.n.u1Present);
4221 Assert(!(pCtx->ss.Attr.u & 0xf00));
4222 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4223 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4224 || !(pCtx->ss.Attr.n.u1Granularity));
4225 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4226 || (pCtx->ss.Attr.n.u1Granularity));
4227 }
4228 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4229 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4230 {
4231 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4232 Assert(pCtx->ds.Attr.n.u1Present);
4233 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4234 Assert(!(pCtx->ds.Attr.u & 0xf00));
4235 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4236 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4237 || !(pCtx->ds.Attr.n.u1Granularity));
4238 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4239 || (pCtx->ds.Attr.n.u1Granularity));
4240 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4241 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4242 }
4243 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4244 {
4245 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4246 Assert(pCtx->es.Attr.n.u1Present);
4247 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4248 Assert(!(pCtx->es.Attr.u & 0xf00));
4249 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4250 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4251 || !(pCtx->es.Attr.n.u1Granularity));
4252 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4253 || (pCtx->es.Attr.n.u1Granularity));
4254 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4255 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4256 }
4257 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4258 {
4259 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4260 Assert(pCtx->fs.Attr.n.u1Present);
4261 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4262 Assert(!(pCtx->fs.Attr.u & 0xf00));
4263 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4264 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4265 || !(pCtx->fs.Attr.n.u1Granularity));
4266 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4267 || (pCtx->fs.Attr.n.u1Granularity));
4268 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4269 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4270 }
4271 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4272 {
4273 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4274 Assert(pCtx->gs.Attr.n.u1Present);
4275 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4276 Assert(!(pCtx->gs.Attr.u & 0xf00));
4277 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4278 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4279 || !(pCtx->gs.Attr.n.u1Granularity));
4280 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4281 || (pCtx->gs.Attr.n.u1Granularity));
4282 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4283 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4284 }
4285 /* 64-bit capable CPUs. */
4286# if HC_ARCH_BITS == 64
4287 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4288 Assert(!pCtx->ss.Attr.u || !RT_HI_U32(pCtx->ss.u64Base));
4289 Assert(!pCtx->ds.Attr.u || !RT_HI_U32(pCtx->ds.u64Base));
4290 Assert(!pCtx->es.Attr.u || !RT_HI_U32(pCtx->es.u64Base));
4291# endif
4292 }
4293 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4294 || ( CPUMIsGuestInRealModeEx(pCtx)
4295 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4296 {
4297 /* Real and v86 mode checks. */
4298 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4299 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4300 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4301 {
4302 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4303 }
4304 else
4305 {
4306 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4307 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4308 }
4309
4310 /* CS */
4311 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4312 Assert(pCtx->cs.u32Limit == 0xffff);
4313 Assert(u32CSAttr == 0xf3);
4314 /* SS */
4315 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4316 Assert(pCtx->ss.u32Limit == 0xffff);
4317 Assert(u32SSAttr == 0xf3);
4318 /* DS */
4319 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4320 Assert(pCtx->ds.u32Limit == 0xffff);
4321 Assert(u32DSAttr == 0xf3);
4322 /* ES */
4323 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4324 Assert(pCtx->es.u32Limit == 0xffff);
4325 Assert(u32ESAttr == 0xf3);
4326 /* FS */
4327 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4328 Assert(pCtx->fs.u32Limit == 0xffff);
4329 Assert(u32FSAttr == 0xf3);
4330 /* GS */
4331 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4332 Assert(pCtx->gs.u32Limit == 0xffff);
4333 Assert(u32GSAttr == 0xf3);
4334 /* 64-bit capable CPUs. */
4335# if HC_ARCH_BITS == 64
4336 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4337 Assert(!u32SSAttr || !RT_HI_U32(pCtx->ss.u64Base));
4338 Assert(!u32DSAttr || !RT_HI_U32(pCtx->ds.u64Base));
4339 Assert(!u32ESAttr || !RT_HI_U32(pCtx->es.u64Base));
4340# endif
4341 }
4342}
4343#endif /* VBOX_STRICT */
4344
4345
4346/**
4347 * Writes a guest segment register into the guest-state area in the VMCS.
4348 *
4349 * @returns VBox status code.
4350 * @param pVCpu The cross context virtual CPU structure.
4351 * @param idxSel Index of the selector in the VMCS.
4352 * @param idxLimit Index of the segment limit in the VMCS.
4353 * @param idxBase Index of the segment base in the VMCS.
4354 * @param idxAccess Index of the access rights of the segment in the VMCS.
4355 * @param pSelReg Pointer to the segment selector.
4356 *
4357 * @remarks No-long-jump zone!!!
4358 */
4359static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4360 uint32_t idxAccess, PCCPUMSELREG pSelReg)
4361{
4362 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4363 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4364 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4365 AssertRCReturn(rc, rc);
4366
4367 uint32_t u32Access = pSelReg->Attr.u;
4368 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4369 {
4370 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4371 u32Access = 0xf3;
4372 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4373 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4374 }
4375 else
4376 {
4377 /*
4378 * The way to differentiate between whether this is really a null selector or was just
4379 * a selector loaded with 0 in real-mode is using the segment attributes. A selector
4380 * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4381 * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4382 * NULL selectors loaded in protected-mode have their attribute as 0.
4383 */
4384 if (!u32Access)
4385 u32Access = X86DESCATTR_UNUSABLE;
4386 }
4387
4388 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4389 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4390 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4391
4392 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4393 AssertRCReturn(rc, rc);
4394 return rc;
4395}
4396
4397
4398/**
4399 * Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4400 * into the guest-state area in the VMCS.
4401 *
4402 * @returns VBox status code.
4403 * @param pVCpu The cross context virtual CPU structure.
4404 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4405 * out-of-sync. Make sure to update the required fields
4406 * before using them.
4407 *
4408 * @remarks Will import guest CR0 on strict builds during validation of
4409 * segments.
4410 * @remarks No-long-jump zone!!!
4411 */
4412static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4413{
4414 int rc = VERR_INTERNAL_ERROR_5;
4415 PVM pVM = pVCpu->CTX_SUFF(pVM);
4416
4417 /*
4418 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4419 */
4420 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4421 {
4422 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4423 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4424 {
4425 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4426 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4427 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4428 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4429 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4430 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4431 }
4432
4433#ifdef VBOX_WITH_REM
4434 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4435 {
4436 Assert(pVM->hm.s.vmx.pRealModeTSS);
4437 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4438 if ( pVCpu->hm.s.vmx.fWasInRealMode
4439 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4440 {
4441 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4442 in real-mode (e.g. OpenBSD 4.0) */
4443 REMFlushTBs(pVM);
4444 Log4Func(("Switch to protected mode detected!\n"));
4445 pVCpu->hm.s.vmx.fWasInRealMode = false;
4446 }
4447 }
4448#endif
4449 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4450 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4451 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4452 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4453 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4454 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4455 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4456 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4457 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4458 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4459 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4460 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4461 AssertRCReturn(rc, rc);
4462
4463#ifdef VBOX_STRICT
4464 hmR0VmxValidateSegmentRegs(pVCpu, pMixedCtx);
4465#endif
4466
4467 /* Update the exit history entry with the correct CS.BASE + RIP. */
4468 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4469 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
4470
4471 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SREG_MASK);
4472 Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4473 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4474 }
4475
4476 /*
4477 * Guest TR.
4478 */
4479 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4480 {
4481 /*
4482 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
4483 * achieved using the interrupt redirection bitmap (all bits cleared to let the guest
4484 * handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4485 */
4486 uint16_t u16Sel = 0;
4487 uint32_t u32Limit = 0;
4488 uint64_t u64Base = 0;
4489 uint32_t u32AccessRights = 0;
4490
4491 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4492 {
4493 u16Sel = pMixedCtx->tr.Sel;
4494 u32Limit = pMixedCtx->tr.u32Limit;
4495 u64Base = pMixedCtx->tr.u64Base;
4496 u32AccessRights = pMixedCtx->tr.Attr.u;
4497 }
4498 else
4499 {
4500 Assert(pVM->hm.s.vmx.pRealModeTSS);
4501 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4502
4503 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4504 RTGCPHYS GCPhys;
4505 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4506 AssertRCReturn(rc, rc);
4507
4508 X86DESCATTR DescAttr;
4509 DescAttr.u = 0;
4510 DescAttr.n.u1Present = 1;
4511 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4512
4513 u16Sel = 0;
4514 u32Limit = HM_VTX_TSS_SIZE;
4515 u64Base = GCPhys; /* in real-mode phys = virt. */
4516 u32AccessRights = DescAttr.u;
4517 }
4518
4519 /* Validate. */
4520 Assert(!(u16Sel & RT_BIT(2)));
4521 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4522 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4523 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4524 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4525 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4526 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4527 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4528 Assert( (u32Limit & 0xfff) == 0xfff
4529 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4530 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4531 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4532
4533 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4534 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4535 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4536 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4537 AssertRCReturn(rc, rc);
4538
4539 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4540 Log4Func(("TR base=%#RX64\n", pMixedCtx->tr.u64Base));
4541 }
4542
4543 /*
4544 * Guest GDTR.
4545 */
4546 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4547 {
4548 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4549 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4550 AssertRCReturn(rc, rc);
4551
4552 /* Validate. */
4553 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4554
4555 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4556 Log4Func(("GDTR base=%#RX64\n", pMixedCtx->gdtr.pGdt));
4557 }
4558
4559 /*
4560 * Guest LDTR.
4561 */
4562 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4563 {
4564 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4565 uint32_t u32Access = 0;
4566 if (!pMixedCtx->ldtr.Attr.u)
4567 u32Access = X86DESCATTR_UNUSABLE;
4568 else
4569 u32Access = pMixedCtx->ldtr.Attr.u;
4570
4571 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4572 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4573 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4574 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4575 AssertRCReturn(rc, rc);
4576
4577 /* Validate. */
4578 if (!(u32Access & X86DESCATTR_UNUSABLE))
4579 {
4580 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4581 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4582 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4583 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4584 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4585 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4586 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4587 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4588 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4589 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4590 }
4591
4592 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4593 Log4Func(("LDTR base=%#RX64\n", pMixedCtx->ldtr.u64Base));
4594 }
4595
4596 /*
4597 * Guest IDTR.
4598 */
4599 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4600 {
4601 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4602 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4603 AssertRCReturn(rc, rc);
4604
4605 /* Validate. */
4606 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4607
4608 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4609 Log4Func(("IDTR base=%#RX64\n", pMixedCtx->idtr.pIdt));
4610 }
4611
4612 return VINF_SUCCESS;
4613}
4614
4615
4616/**
4617 * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4618 * areas.
4619 *
4620 * These MSRs will automatically be loaded to the host CPU on every successful
4621 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4622 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4623 * -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4624 *
4625 * Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4626 *
4627 * @returns VBox status code.
4628 * @param pVCpu The cross context virtual CPU structure.
4629 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4630 * out-of-sync. Make sure to update the required fields
4631 * before using them.
4632 *
4633 * @remarks No-long-jump zone!!!
4634 */
4635static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4636{
4637 AssertPtr(pVCpu);
4638 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4639
4640 /*
4641 * MSRs that we use the auto-load/store MSR area in the VMCS.
4642 * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4643 */
4644 PVM pVM = pVCpu->CTX_SUFF(pVM);
4645 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4646 {
4647 if (pVM->hm.s.fAllow64BitGuests)
4648 {
4649#if HC_ARCH_BITS == 32
4650 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4651 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4652 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4653 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4654 AssertRCReturn(rc, rc);
4655# ifdef LOG_ENABLED
4656 PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4657 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4658 Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4659# endif
4660#endif
4661 }
4662 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4663 }
4664
4665 /*
4666 * Guest Sysenter MSRs.
4667 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4668 * VM-exits on WRMSRs for these MSRs.
4669 */
4670 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4671 {
4672 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4673 {
4674 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs);
4675 AssertRCReturn(rc, rc);
4676 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4677 }
4678
4679 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4680 {
4681 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip);
4682 AssertRCReturn(rc, rc);
4683 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4684 }
4685
4686 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4687 {
4688 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp);
4689 AssertRCReturn(rc, rc);
4690 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4691 }
4692 }
4693
4694 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4695 {
4696 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4697 {
4698 /*
4699 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4700 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4701 */
4702 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4703 {
4704 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4705 AssertRCReturn(rc,rc);
4706 Log4Func(("EFER=%#RX64\n", pMixedCtx->msrEFER));
4707 }
4708 else
4709 {
4710 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4711 NULL /* pfAddedAndUpdated */);
4712 AssertRCReturn(rc, rc);
4713
4714 /* We need to intercept reads too, see @bugref{7386#c16}. */
4715 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4716 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4717 Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pMixedCtx->msrEFER,
4718 pVCpu->hm.s.vmx.cMsrs));
4719 }
4720 }
4721 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4722 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4723 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4724 }
4725
4726 return VINF_SUCCESS;
4727}
4728
4729
4730#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4731/**
4732 * Check if guest state allows safe use of 32-bit switcher again.
4733 *
4734 * Segment bases and protected mode structures must be 32-bit addressable
4735 * because the 32-bit switcher will ignore high dword when writing these VMCS
4736 * fields. See @bugref{8432} for details.
4737 *
4738 * @returns true if safe, false if must continue to use the 64-bit switcher.
4739 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4740 * out-of-sync. Make sure to update the required fields
4741 * before using them.
4742 *
4743 * @remarks No-long-jump zone!!!
4744 */
4745static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pMixedCtx)
4746{
4747 if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4748 if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4749 if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4750 if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4751 if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4752 if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4753 if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4754 if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4755 if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4756 if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4757
4758 /* All good, bases are 32-bit. */
4759 return true;
4760}
4761#endif
4762
4763
4764/**
4765 * Selects up the appropriate function to run guest code.
4766 *
4767 * @returns VBox status code.
4768 * @param pVCpu The cross context virtual CPU structure.
4769 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4770 * out-of-sync. Make sure to update the required fields
4771 * before using them.
4772 *
4773 * @remarks No-long-jump zone!!!
4774 */
4775static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4776{
4777 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4778 {
4779#ifndef VBOX_ENABLE_64_BITS_GUESTS
4780 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4781#endif
4782 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4783#if HC_ARCH_BITS == 32
4784 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4785 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4786 {
4787#ifdef VBOX_STRICT
4788 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4789 {
4790 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4791 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4792 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4793 | HM_CHANGED_VMX_ENTRY_CTLS
4794 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4795 }
4796#endif
4797 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4798
4799 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4800 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4801 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4802 Log4Func(("Selected 64-bit switcher\n"));
4803 }
4804#else
4805 /* 64-bit host. */
4806 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4807#endif
4808 }
4809 else
4810 {
4811 /* Guest is not in long mode, use the 32-bit handler. */
4812#if HC_ARCH_BITS == 32
4813 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4814 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4815 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4816 {
4817# ifdef VBOX_STRICT
4818 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4819 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4820 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4821 | HM_CHANGED_VMX_ENTRY_CTLS
4822 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4823# endif
4824 }
4825# ifdef VBOX_ENABLE_64_BITS_GUESTS
4826 /*
4827 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4828 * design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4829 * switcher flag because now we know the guest is in a sane state where it's safe
4830 * to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4831 * the much faster 32-bit switcher again.
4832 */
4833 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4834 {
4835 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4836 Log4Func(("Selected 32-bit switcher\n"));
4837 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4838 }
4839 else
4840 {
4841 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4842 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4843 || hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
4844 {
4845 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4846 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4847 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4848 | HM_CHANGED_VMX_ENTRY_CTLS
4849 | HM_CHANGED_VMX_EXIT_CTLS
4850 | HM_CHANGED_HOST_CONTEXT);
4851 Log4Func(("Selected 32-bit switcher (safe)\n"));
4852 }
4853 }
4854# else
4855 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4856# endif
4857#else
4858 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4859#endif
4860 }
4861 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4862 return VINF_SUCCESS;
4863}
4864
4865
4866/**
4867 * Wrapper for running the guest code in VT-x.
4868 *
4869 * @returns VBox status code, no informational status codes.
4870 * @param pVCpu The cross context virtual CPU structure.
4871 * @param pCtx Pointer to the guest-CPU context.
4872 *
4873 * @remarks No-long-jump zone!!!
4874 */
4875DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PCPUMCTX pCtx)
4876{
4877 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4878 pCtx->fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
4879
4880 /*
4881 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4882 * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4883 * callee-saved and thus the need for this XMM wrapper.
4884 *
4885 * See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4886 */
4887 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4888 /** @todo Add stats for resume vs launch. */
4889 PVM pVM = pVCpu->CTX_SUFF(pVM);
4890#ifdef VBOX_WITH_KERNEL_USING_XMM
4891 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4892#else
4893 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4894#endif
4895 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4896 return rc;
4897}
4898
4899
4900/**
4901 * Reports world-switch error and dumps some useful debug info.
4902 *
4903 * @param pVCpu The cross context virtual CPU structure.
4904 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4905 * @param pCtx Pointer to the guest-CPU context.
4906 * @param pVmxTransient Pointer to the VMX transient structure (only
4907 * exitReason updated).
4908 */
4909static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4910{
4911 Assert(pVCpu);
4912 Assert(pCtx);
4913 Assert(pVmxTransient);
4914 HMVMX_ASSERT_PREEMPT_SAFE();
4915
4916 Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
4917 switch (rcVMRun)
4918 {
4919 case VERR_VMX_INVALID_VMXON_PTR:
4920 AssertFailed();
4921 break;
4922 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4923 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4924 {
4925 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4926 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4927 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4928 AssertRC(rc);
4929
4930 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4931 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4932 Cannot do it here as we may have been long preempted. */
4933
4934#ifdef VBOX_STRICT
4935 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4936 pVmxTransient->uExitReason));
4937 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4938 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4939 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4940 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4941 else
4942 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4943 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4944 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4945
4946 /* VMX control bits. */
4947 uint32_t u32Val;
4948 uint64_t u64Val;
4949 RTHCUINTREG uHCReg;
4950 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4951 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4952 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4953 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4954 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
4955 {
4956 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4957 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4958 }
4959 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4960 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4961 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4962 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4963 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4964 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4965 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4966 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4967 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4968 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4969 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4970 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4971 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4972 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4973 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4974 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4975 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4976 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4977 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4978 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4979 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4980 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4981 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4982 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4983 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4984 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4985 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4986 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4987 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4988 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4989 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4990 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4991 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4992 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4993 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
4994 {
4995 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4996 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4997 }
4998
4999 /* Guest bits. */
5000 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5001 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5002 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5003 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5004 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5005 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5006 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
5007 {
5008 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5009 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5010 }
5011
5012 /* Host bits. */
5013 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5014 Log4(("Host CR0 %#RHr\n", uHCReg));
5015 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5016 Log4(("Host CR3 %#RHr\n", uHCReg));
5017 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5018 Log4(("Host CR4 %#RHr\n", uHCReg));
5019
5020 RTGDTR HostGdtr;
5021 PCX86DESCHC pDesc;
5022 ASMGetGDTR(&HostGdtr);
5023 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5024 Log4(("Host CS %#08x\n", u32Val));
5025 if (u32Val < HostGdtr.cbGdt)
5026 {
5027 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5028 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5029 }
5030
5031 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5032 Log4(("Host DS %#08x\n", u32Val));
5033 if (u32Val < HostGdtr.cbGdt)
5034 {
5035 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5036 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5037 }
5038
5039 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5040 Log4(("Host ES %#08x\n", u32Val));
5041 if (u32Val < HostGdtr.cbGdt)
5042 {
5043 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5044 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5045 }
5046
5047 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5048 Log4(("Host FS %#08x\n", u32Val));
5049 if (u32Val < HostGdtr.cbGdt)
5050 {
5051 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5052 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5053 }
5054
5055 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5056 Log4(("Host GS %#08x\n", u32Val));
5057 if (u32Val < HostGdtr.cbGdt)
5058 {
5059 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5060 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5061 }
5062
5063 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5064 Log4(("Host SS %#08x\n", u32Val));
5065 if (u32Val < HostGdtr.cbGdt)
5066 {
5067 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5068 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5069 }
5070
5071 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5072 Log4(("Host TR %#08x\n", u32Val));
5073 if (u32Val < HostGdtr.cbGdt)
5074 {
5075 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5076 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5077 }
5078
5079 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5080 Log4(("Host TR Base %#RHv\n", uHCReg));
5081 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5082 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5083 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5084 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5085 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5086 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5087 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5088 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5089 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5090 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5091 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5092 Log4(("Host RSP %#RHv\n", uHCReg));
5093 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5094 Log4(("Host RIP %#RHv\n", uHCReg));
5095# if HC_ARCH_BITS == 64
5096 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5097 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5098 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5099 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5100 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5101 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5102# endif
5103#endif /* VBOX_STRICT */
5104 break;
5105 }
5106
5107 default:
5108 /* Impossible */
5109 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5110 break;
5111 }
5112 NOREF(pCtx);
5113}
5114
5115
5116#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5117#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5118# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5119#endif
5120#ifdef VBOX_STRICT
5121static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5122{
5123 switch (idxField)
5124 {
5125 case VMX_VMCS_GUEST_RIP:
5126 case VMX_VMCS_GUEST_RSP:
5127 case VMX_VMCS_GUEST_SYSENTER_EIP:
5128 case VMX_VMCS_GUEST_SYSENTER_ESP:
5129 case VMX_VMCS_GUEST_GDTR_BASE:
5130 case VMX_VMCS_GUEST_IDTR_BASE:
5131 case VMX_VMCS_GUEST_CS_BASE:
5132 case VMX_VMCS_GUEST_DS_BASE:
5133 case VMX_VMCS_GUEST_ES_BASE:
5134 case VMX_VMCS_GUEST_FS_BASE:
5135 case VMX_VMCS_GUEST_GS_BASE:
5136 case VMX_VMCS_GUEST_SS_BASE:
5137 case VMX_VMCS_GUEST_LDTR_BASE:
5138 case VMX_VMCS_GUEST_TR_BASE:
5139 case VMX_VMCS_GUEST_CR3:
5140 return true;
5141 }
5142 return false;
5143}
5144
5145static bool hmR0VmxIsValidReadField(uint32_t idxField)
5146{
5147 switch (idxField)
5148 {
5149 /* Read-only fields. */
5150 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5151 return true;
5152 }
5153 /* Remaining readable fields should also be writable. */
5154 return hmR0VmxIsValidWriteField(idxField);
5155}
5156#endif /* VBOX_STRICT */
5157
5158
5159/**
5160 * Executes the specified handler in 64-bit mode.
5161 *
5162 * @returns VBox status code (no informational status codes).
5163 * @param pVCpu The cross context virtual CPU structure.
5164 * @param enmOp The operation to perform.
5165 * @param cParams Number of parameters.
5166 * @param paParam Array of 32-bit parameters.
5167 */
5168VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp,
5169 uint32_t cParams, uint32_t *paParam)
5170{
5171 PVM pVM = pVCpu->CTX_SUFF(pVM);
5172 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5173 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5174 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5175 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5176
5177#ifdef VBOX_STRICT
5178 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5179 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5180
5181 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5182 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5183#endif
5184
5185 /* Disable interrupts. */
5186 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5187
5188#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5189 RTCPUID idHostCpu = RTMpCpuId();
5190 CPUMR0SetLApic(pVCpu, idHostCpu);
5191#endif
5192
5193 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5194 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5195
5196 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5197 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5198 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5199
5200 /* Leave VMX Root Mode. */
5201 VMXDisable();
5202
5203 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5204
5205 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5206 CPUMSetHyperEIP(pVCpu, enmOp);
5207 for (int i = (int)cParams - 1; i >= 0; i--)
5208 CPUMPushHyper(pVCpu, paParam[i]);
5209
5210 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5211
5212 /* Call the switcher. */
5213 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5214 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5215
5216 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5217 /* Make sure the VMX instructions don't cause #UD faults. */
5218 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5219
5220 /* Re-enter VMX Root Mode */
5221 int rc2 = VMXEnable(HCPhysCpuPage);
5222 if (RT_FAILURE(rc2))
5223 {
5224 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5225 ASMSetFlags(fOldEFlags);
5226 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5227 return rc2;
5228 }
5229
5230 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5231 AssertRC(rc2);
5232 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5233 Assert(!(ASMGetFlags() & X86_EFL_IF));
5234 ASMSetFlags(fOldEFlags);
5235 return rc;
5236}
5237
5238
5239/**
5240 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5241 * supporting 64-bit guests.
5242 *
5243 * @returns VBox status code.
5244 * @param fResume Whether to VMLAUNCH or VMRESUME.
5245 * @param pCtx Pointer to the guest-CPU context.
5246 * @param pCache Pointer to the VMCS cache.
5247 * @param pVM The cross context VM structure.
5248 * @param pVCpu The cross context virtual CPU structure.
5249 */
5250DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5251{
5252 NOREF(fResume);
5253
5254 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5255 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5256
5257#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5258 pCache->uPos = 1;
5259 pCache->interPD = PGMGetInterPaeCR3(pVM);
5260 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5261#endif
5262
5263#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5264 pCache->TestIn.HCPhysCpuPage = 0;
5265 pCache->TestIn.HCPhysVmcs = 0;
5266 pCache->TestIn.pCache = 0;
5267 pCache->TestOut.HCPhysVmcs = 0;
5268 pCache->TestOut.pCache = 0;
5269 pCache->TestOut.pCtx = 0;
5270 pCache->TestOut.eflags = 0;
5271#else
5272 NOREF(pCache);
5273#endif
5274
5275 uint32_t aParam[10];
5276 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5277 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5278 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5279 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5280 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5281 aParam[5] = 0;
5282 aParam[6] = VM_RC_ADDR(pVM, pVM);
5283 aParam[7] = 0;
5284 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5285 aParam[9] = 0;
5286
5287#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5288 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5289 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5290#endif
5291 int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5292
5293#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5294 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5295 Assert(pCtx->dr[4] == 10);
5296 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5297#endif
5298
5299#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5300 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5301 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5302 pVCpu->hm.s.vmx.HCPhysVmcs));
5303 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5304 pCache->TestOut.HCPhysVmcs));
5305 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5306 pCache->TestOut.pCache));
5307 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5308 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5309 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5310 pCache->TestOut.pCtx));
5311 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5312#endif
5313 NOREF(pCtx);
5314 return rc;
5315}
5316
5317
5318/**
5319 * Initialize the VMCS-Read cache.
5320 *
5321 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5322 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5323 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5324 * (those that have a 32-bit FULL & HIGH part).
5325 *
5326 * @returns VBox status code.
5327 * @param pVCpu The cross context virtual CPU structure.
5328 */
5329static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
5330{
5331#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5332 do { \
5333 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5334 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5335 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5336 ++cReadFields; \
5337 } while (0)
5338
5339 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5340 uint32_t cReadFields = 0;
5341
5342 /*
5343 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5344 * and serve to indicate exceptions to the rules.
5345 */
5346
5347 /* Guest-natural selector base fields. */
5348#if 0
5349 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5350 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5351 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5352#endif
5353 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5354 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5355 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5356 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5357 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5358 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5359 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5360 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5361 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5362 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5363 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5364 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5365#if 0
5366 /* Unused natural width guest-state fields. */
5367 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5368 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5369#endif
5370 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5371 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5372
5373 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5374 these 64-bit fields (using "FULL" and "HIGH" fields). */
5375#if 0
5376 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5377 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5378 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5379 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5380 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5381 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5382 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5383 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5384 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5385#endif
5386
5387 /* Natural width guest-state fields. */
5388 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5389#if 0
5390 /* Currently unused field. */
5391 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5392#endif
5393
5394 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5395 {
5396 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5397 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5398 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5399 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5400 }
5401 else
5402 {
5403 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5404 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5405 }
5406
5407#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5408 return VINF_SUCCESS;
5409}
5410
5411
5412/**
5413 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5414 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5415 * darwin, running 64-bit guests).
5416 *
5417 * @returns VBox status code.
5418 * @param pVCpu The cross context virtual CPU structure.
5419 * @param idxField The VMCS field encoding.
5420 * @param u64Val 16, 32 or 64-bit value.
5421 */
5422VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5423{
5424 int rc;
5425 switch (idxField)
5426 {
5427 /*
5428 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5429 */
5430 /* 64-bit Control fields. */
5431 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5432 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5433 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5434 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5435 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5436 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5437 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5438 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5439 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5440 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5441 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5442 case VMX_VMCS64_CTRL_EPTP_FULL:
5443 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5444 /* 64-bit Guest-state fields. */
5445 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5446 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5447 case VMX_VMCS64_GUEST_PAT_FULL:
5448 case VMX_VMCS64_GUEST_EFER_FULL:
5449 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5450 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5451 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5452 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5453 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5454 /* 64-bit Host-state fields. */
5455 case VMX_VMCS64_HOST_PAT_FULL:
5456 case VMX_VMCS64_HOST_EFER_FULL:
5457 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5458 {
5459 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5460 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5461 break;
5462 }
5463
5464 /*
5465 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5466 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5467 */
5468 /* Natural-width Guest-state fields. */
5469 case VMX_VMCS_GUEST_CR3:
5470 case VMX_VMCS_GUEST_ES_BASE:
5471 case VMX_VMCS_GUEST_CS_BASE:
5472 case VMX_VMCS_GUEST_SS_BASE:
5473 case VMX_VMCS_GUEST_DS_BASE:
5474 case VMX_VMCS_GUEST_FS_BASE:
5475 case VMX_VMCS_GUEST_GS_BASE:
5476 case VMX_VMCS_GUEST_LDTR_BASE:
5477 case VMX_VMCS_GUEST_TR_BASE:
5478 case VMX_VMCS_GUEST_GDTR_BASE:
5479 case VMX_VMCS_GUEST_IDTR_BASE:
5480 case VMX_VMCS_GUEST_RSP:
5481 case VMX_VMCS_GUEST_RIP:
5482 case VMX_VMCS_GUEST_SYSENTER_ESP:
5483 case VMX_VMCS_GUEST_SYSENTER_EIP:
5484 {
5485 if (!(RT_HI_U32(u64Val)))
5486 {
5487 /* If this field is 64-bit, VT-x will zero out the top bits. */
5488 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5489 }
5490 else
5491 {
5492 /* Assert that only the 32->64 switcher case should ever come here. */
5493 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5494 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5495 }
5496 break;
5497 }
5498
5499 default:
5500 {
5501 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5502 rc = VERR_INVALID_PARAMETER;
5503 break;
5504 }
5505 }
5506 AssertRCReturn(rc, rc);
5507 return rc;
5508}
5509
5510
5511/**
5512 * Queue up a VMWRITE by using the VMCS write cache.
5513 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5514 *
5515 * @param pVCpu The cross context virtual CPU structure.
5516 * @param idxField The VMCS field encoding.
5517 * @param u64Val 16, 32 or 64-bit value.
5518 */
5519VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5520{
5521 AssertPtr(pVCpu);
5522 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5523
5524 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5525 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5526
5527 /* Make sure there are no duplicates. */
5528 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5529 {
5530 if (pCache->Write.aField[i] == idxField)
5531 {
5532 pCache->Write.aFieldVal[i] = u64Val;
5533 return VINF_SUCCESS;
5534 }
5535 }
5536
5537 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5538 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5539 pCache->Write.cValidEntries++;
5540 return VINF_SUCCESS;
5541}
5542#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5543
5544
5545/**
5546 * Sets up the usage of TSC-offsetting and updates the VMCS.
5547 *
5548 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5549 * VMX preemption timer.
5550 *
5551 * @returns VBox status code.
5552 * @param pVCpu The cross context virtual CPU structure.
5553 *
5554 * @remarks No-long-jump zone!!!
5555 */
5556static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5557{
5558 int rc;
5559 bool fOffsettedTsc;
5560 bool fParavirtTsc;
5561 PVM pVM = pVCpu->CTX_SUFF(pVM);
5562 if (pVM->hm.s.vmx.fUsePreemptTimer)
5563 {
5564 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5565 &fOffsettedTsc, &fParavirtTsc);
5566
5567 /* Make sure the returned values have sane upper and lower boundaries. */
5568 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5569 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5570 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5571 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5572
5573 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5574 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5575 }
5576 else
5577 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5578
5579 /** @todo later optimize this to be done elsewhere and not before every
5580 * VM-entry. */
5581 if (fParavirtTsc)
5582 {
5583 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5584 information before every VM-entry, hence disable it for performance sake. */
5585#if 0
5586 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5587 AssertRC(rc);
5588#endif
5589 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5590 }
5591
5592 if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5593 {
5594 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5595 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5596
5597 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5598 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5599 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5600 }
5601 else
5602 {
5603 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5604 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5605 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5606 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5607 }
5608}
5609
5610
5611/**
5612 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5613 * VM-exit interruption info type.
5614 *
5615 * @returns The IEM exception flags.
5616 * @param uVector The event vector.
5617 * @param uVmxVectorType The VMX event type.
5618 *
5619 * @remarks This function currently only constructs flags required for
5620 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5621 * and CR2 aspects of an exception are not included).
5622 */
5623static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5624{
5625 uint32_t fIemXcptFlags;
5626 switch (uVmxVectorType)
5627 {
5628 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5629 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5630 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5631 break;
5632
5633 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5634 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5635 break;
5636
5637 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5638 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5639 break;
5640
5641 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5642 {
5643 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5644 if (uVector == X86_XCPT_BP)
5645 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5646 else if (uVector == X86_XCPT_OF)
5647 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5648 else
5649 {
5650 fIemXcptFlags = 0;
5651 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5652 }
5653 break;
5654 }
5655
5656 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5657 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5658 break;
5659
5660 default:
5661 fIemXcptFlags = 0;
5662 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5663 break;
5664 }
5665 return fIemXcptFlags;
5666}
5667
5668
5669/**
5670 * Sets an event as a pending event to be injected into the guest.
5671 *
5672 * @param pVCpu The cross context virtual CPU structure.
5673 * @param u32IntInfo The VM-entry interruption-information field.
5674 * @param cbInstr The VM-entry instruction length in bytes (for software
5675 * interrupts, exceptions and privileged software
5676 * exceptions).
5677 * @param u32ErrCode The VM-entry exception error code.
5678 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5679 * page-fault.
5680 *
5681 * @remarks Statistics counter assumes this is a guest event being injected or
5682 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5683 * always incremented.
5684 */
5685DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5686 RTGCUINTPTR GCPtrFaultAddress)
5687{
5688 Assert(!pVCpu->hm.s.Event.fPending);
5689 pVCpu->hm.s.Event.fPending = true;
5690 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5691 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5692 pVCpu->hm.s.Event.cbInstr = cbInstr;
5693 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5694}
5695
5696
5697/**
5698 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5699 *
5700 * @param pVCpu The cross context virtual CPU structure.
5701 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5702 * out-of-sync. Make sure to update the required fields
5703 * before using them.
5704 */
5705DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5706{
5707 NOREF(pMixedCtx);
5708 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5709 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5710 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5711 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5712}
5713
5714
5715/**
5716 * Handle a condition that occurred while delivering an event through the guest
5717 * IDT.
5718 *
5719 * @returns Strict VBox status code (i.e. informational status codes too).
5720 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5721 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5722 * to continue execution of the guest which will delivery the \#DF.
5723 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5724 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5725 *
5726 * @param pVCpu The cross context virtual CPU structure.
5727 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5728 * out-of-sync. Make sure to update the required fields
5729 * before using them.
5730 * @param pVmxTransient Pointer to the VMX transient structure.
5731 *
5732 * @remarks No-long-jump zone!!!
5733 */
5734static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5735{
5736 uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5737
5738 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5739 rc2 |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5740 AssertRCReturn(rc2, rc2);
5741
5742 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5743 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5744 {
5745 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5746 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5747
5748 /*
5749 * If the event was a software interrupt (generated with INT n) or a software exception
5750 * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
5751 * can handle the VM-exit and continue guest execution which will re-execute the
5752 * instruction rather than re-injecting the exception, as that can cause premature
5753 * trips to ring-3 before injection and involve TRPM which currently has no way of
5754 * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
5755 * the problem).
5756 */
5757 IEMXCPTRAISE enmRaise;
5758 IEMXCPTRAISEINFO fRaiseInfo;
5759 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5760 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5761 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5762 {
5763 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5764 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5765 }
5766 else if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5767 {
5768 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5769 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5770 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5771 /** @todo Make AssertMsgReturn as just AssertMsg later. */
5772 AssertMsgReturn(uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT,
5773 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5774 uExitVectorType), VERR_VMX_IPE_5);
5775
5776 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5777
5778 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5779 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
5780 {
5781 pVmxTransient->fVectoringPF = true;
5782 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5783 }
5784 }
5785 else
5786 {
5787 /*
5788 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5789 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5790 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
5791 */
5792 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5793 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5794 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5795 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5796 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5797 }
5798
5799 /*
5800 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5801 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5802 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5803 * subsequent VM-entry would fail.
5804 *
5805 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5806 */
5807 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5808 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5809 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
5810 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
5811 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5812 {
5813 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5814 }
5815
5816 switch (enmRaise)
5817 {
5818 case IEMXCPTRAISE_CURRENT_XCPT:
5819 {
5820 Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
5821 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
5822 Assert(rcStrict == VINF_SUCCESS);
5823 break;
5824 }
5825
5826 case IEMXCPTRAISE_PREV_EVENT:
5827 {
5828 uint32_t u32ErrCode;
5829 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5830 {
5831 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5832 AssertRCReturn(rc2, rc2);
5833 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5834 }
5835 else
5836 u32ErrCode = 0;
5837
5838 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
5839 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5840 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5841 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5842
5843 Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
5844 pVCpu->hm.s.Event.u32ErrCode));
5845 Assert(rcStrict == VINF_SUCCESS);
5846 break;
5847 }
5848
5849 case IEMXCPTRAISE_REEXEC_INSTR:
5850 Assert(rcStrict == VINF_SUCCESS);
5851 break;
5852
5853 case IEMXCPTRAISE_DOUBLE_FAULT:
5854 {
5855 /*
5856 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
5857 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
5858 */
5859 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
5860 {
5861 pVmxTransient->fVectoringDoublePF = true;
5862 Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
5863 pMixedCtx->cr2));
5864 rcStrict = VINF_SUCCESS;
5865 }
5866 else
5867 {
5868 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5869 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5870 Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
5871 uIdtVector, uExitVector));
5872 rcStrict = VINF_HM_DOUBLE_FAULT;
5873 }
5874 break;
5875 }
5876
5877 case IEMXCPTRAISE_TRIPLE_FAULT:
5878 {
5879 Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
5880 rcStrict = VINF_EM_RESET;
5881 break;
5882 }
5883
5884 case IEMXCPTRAISE_CPU_HANG:
5885 {
5886 Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
5887 rcStrict = VERR_EM_GUEST_CPU_HANG;
5888 break;
5889 }
5890
5891 default:
5892 {
5893 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
5894 rcStrict = VERR_VMX_IPE_2;
5895 break;
5896 }
5897 }
5898 }
5899 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5900 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5901 && uExitVector != X86_XCPT_DF
5902 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5903 {
5904 /*
5905 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5906 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5907 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5908 */
5909 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5910 {
5911 Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
5912 VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5913 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5914 }
5915 }
5916
5917 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
5918 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
5919 return rcStrict;
5920}
5921
5922
5923/**
5924 * Imports a guest segment register from the current VMCS into
5925 * the guest-CPU context.
5926 *
5927 * @returns VBox status code.
5928 * @param pVCpu The cross context virtual CPU structure.
5929 * @param idxSel Index of the selector in the VMCS.
5930 * @param idxLimit Index of the segment limit in the VMCS.
5931 * @param idxBase Index of the segment base in the VMCS.
5932 * @param idxAccess Index of the access rights of the segment in the VMCS.
5933 * @param pSelReg Pointer to the segment selector.
5934 *
5935 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
5936 * do not log!
5937 *
5938 * @remarks Never call this function directly!!! Use the
5939 * HMVMX_IMPORT_SREG() macro as that takes care
5940 * of whether to read from the VMCS cache or not.
5941 */
5942static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5943 PCPUMSELREG pSelReg)
5944{
5945 NOREF(pVCpu);
5946
5947 uint32_t u32Sel;
5948 uint32_t u32Limit;
5949 uint32_t u32Attr;
5950 uint64_t u64Base;
5951 int rc = VMXReadVmcs32(idxSel, &u32Sel);
5952 rc |= VMXReadVmcs32(idxLimit, &u32Limit);
5953 rc |= VMXReadVmcs32(idxAccess, &u32Attr);
5954 rc |= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
5955 AssertRCReturn(rc, rc);
5956
5957 pSelReg->Sel = (uint16_t)u32Sel;
5958 pSelReg->ValidSel = (uint16_t)u32Sel;
5959 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5960 pSelReg->u32Limit = u32Limit;
5961 pSelReg->u64Base = u64Base;
5962 pSelReg->Attr.u = u32Attr;
5963
5964 /*
5965 * If VT-x marks the segment as unusable, most other bits remain undefined:
5966 * - For CS the L, D and G bits have meaning.
5967 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
5968 * - For the remaining data segments no bits are defined.
5969 *
5970 * The present bit and the unusable bit has been observed to be set at the
5971 * same time (the selector was supposed to be invalid as we started executing
5972 * a V8086 interrupt in ring-0).
5973 *
5974 * What should be important for the rest of the VBox code, is that the P bit is
5975 * cleared. Some of the other VBox code recognizes the unusable bit, but
5976 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
5977 * safe side here, we'll strip off P and other bits we don't care about. If
5978 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
5979 *
5980 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5981 */
5982 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
5983 {
5984 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
5985
5986 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
5987 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
5988 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
5989#ifdef VBOX_STRICT
5990 VMMRZCallRing3Disable(pVCpu);
5991 Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
5992# ifdef DEBUG_bird
5993 AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
5994 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
5995 idxSel, u32Sel, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
5996# endif
5997 VMMRZCallRing3Enable(pVCpu);
5998#endif
5999 }
6000 return VINF_SUCCESS;
6001}
6002
6003
6004/**
6005 * Imports the guest RIP from the VMCS back into the guest-CPU context.
6006 *
6007 * @returns VBox status code.
6008 * @param pVCpu The cross context virtual CPU structure.
6009 *
6010 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6011 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6012 * instead!!!
6013 */
6014DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6015{
6016 uint64_t u64Val;
6017 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6018 if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6019 {
6020 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6021 if (RT_SUCCESS(rc))
6022 {
6023 pCtx->rip = u64Val;
6024 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6025 }
6026 return rc;
6027 }
6028 return VINF_SUCCESS;
6029}
6030
6031
6032/**
6033 * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6034 *
6035 * @returns VBox status code.
6036 * @param pVCpu The cross context virtual CPU structure.
6037 *
6038 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6039 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6040 * instead!!!
6041 */
6042DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6043{
6044 uint32_t u32Val;
6045 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6046 if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6047 {
6048 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6049 if (RT_SUCCESS(rc))
6050 {
6051 pCtx->eflags.u32 = u32Val;
6052
6053 /* Restore eflags for real-on-v86-mode hack. */
6054 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6055 {
6056 pCtx->eflags.Bits.u1VM = 0;
6057 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6058 }
6059 }
6060 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6061 return rc;
6062 }
6063 return VINF_SUCCESS;
6064}
6065
6066
6067/**
6068 * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6069 * context.
6070 *
6071 * @returns VBox status code.
6072 * @param pVCpu The cross context virtual CPU structure.
6073 *
6074 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6075 * do not log!
6076 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6077 * instead!!!
6078 */
6079DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6080{
6081 uint32_t u32Val;
6082 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6083 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32Val);
6084 if (RT_SUCCESS(rc))
6085 {
6086 /*
6087 * We additionally have a requirement to import RIP, RFLAGS depending on whether we
6088 * might need them in hmR0VmxEvaluatePendingEvent().
6089 */
6090 if (!u32Val)
6091 {
6092 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6093 {
6094 rc = hmR0VmxImportGuestRip(pVCpu);
6095 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6096 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6097 }
6098
6099 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6100 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6101 }
6102 else
6103 {
6104 rc = hmR0VmxImportGuestRip(pVCpu);
6105 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6106
6107 if (u32Val & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6108 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6109 {
6110 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6111 }
6112 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6113 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6114
6115 if (u32Val & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6116 {
6117 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6118 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6119 }
6120 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6121 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6122 }
6123 }
6124 return rc;
6125}
6126
6127
6128/**
6129 * Worker for VMXR0ImportStateOnDemand.
6130 *
6131 * @returns VBox status code.
6132 * @param pVCpu The cross context virtual CPU structure.
6133 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6134 */
6135static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6136{
6137#define VMXLOCAL_BREAK_RC(a_rc) \
6138 if (RT_FAILURE(a_rc)) \
6139 break
6140
6141 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6142
6143 int rc = VINF_SUCCESS;
6144 PVM pVM = pVCpu->CTX_SUFF(pVM);
6145 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6146 uint64_t u64Val;
6147 uint32_t u32Val;
6148
6149 Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6150
6151 /*
6152 * We disable interrupts to make the updating of the state and in particular
6153 * the fExtrn modification atomic wrt to preemption hooks.
6154 */
6155 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6156
6157 fWhat &= pCtx->fExtrn;
6158 if (fWhat)
6159 {
6160 do
6161 {
6162 if (fWhat & CPUMCTX_EXTRN_RIP)
6163 {
6164 rc = hmR0VmxImportGuestRip(pVCpu);
6165 VMXLOCAL_BREAK_RC(rc);
6166 }
6167
6168 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6169 {
6170 rc = hmR0VmxImportGuestRFlags(pVCpu);
6171 VMXLOCAL_BREAK_RC(rc);
6172 }
6173
6174 if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6175 {
6176 rc = hmR0VmxImportGuestIntrState(pVCpu);
6177 VMXLOCAL_BREAK_RC(rc);
6178 }
6179
6180 if (fWhat & CPUMCTX_EXTRN_RSP)
6181 {
6182 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6183 VMXLOCAL_BREAK_RC(rc);
6184 pCtx->rsp = u64Val;
6185 }
6186
6187 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6188 {
6189 if (fWhat & CPUMCTX_EXTRN_CS)
6190 {
6191 rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6192 VMXLOCAL_BREAK_RC(rc);
6193 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6194 pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6195 }
6196 if (fWhat & CPUMCTX_EXTRN_SS)
6197 {
6198 rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6199 VMXLOCAL_BREAK_RC(rc);
6200 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6201 pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6202 }
6203 if (fWhat & CPUMCTX_EXTRN_DS)
6204 {
6205 rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6206 VMXLOCAL_BREAK_RC(rc);
6207 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6208 pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6209 }
6210 if (fWhat & CPUMCTX_EXTRN_ES)
6211 {
6212 rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6213 VMXLOCAL_BREAK_RC(rc);
6214 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6215 pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6216 }
6217 if (fWhat & CPUMCTX_EXTRN_FS)
6218 {
6219 rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6220 VMXLOCAL_BREAK_RC(rc);
6221 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6222 pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6223 }
6224 if (fWhat & CPUMCTX_EXTRN_GS)
6225 {
6226 rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6227 VMXLOCAL_BREAK_RC(rc);
6228 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6229 pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6230 }
6231 }
6232
6233 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6234 {
6235 if (fWhat & CPUMCTX_EXTRN_LDTR)
6236 {
6237 rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6238 VMXLOCAL_BREAK_RC(rc);
6239 }
6240
6241 if (fWhat & CPUMCTX_EXTRN_GDTR)
6242 {
6243 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6244 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6245 VMXLOCAL_BREAK_RC(rc);
6246 pCtx->gdtr.pGdt = u64Val;
6247 pCtx->gdtr.cbGdt = u32Val;
6248 }
6249
6250 /* Guest IDTR. */
6251 if (fWhat & CPUMCTX_EXTRN_IDTR)
6252 {
6253 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6254 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6255 VMXLOCAL_BREAK_RC(rc);
6256 pCtx->idtr.pIdt = u64Val;
6257 pCtx->idtr.cbIdt = u32Val;
6258 }
6259
6260 /* Guest TR. */
6261 if (fWhat & CPUMCTX_EXTRN_TR)
6262 {
6263 /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6264 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6265 {
6266 rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6267 VMXLOCAL_BREAK_RC(rc);
6268 }
6269 }
6270 }
6271
6272 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6273 {
6274 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6275 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6276 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6277 pCtx->SysEnter.cs = u32Val;
6278 VMXLOCAL_BREAK_RC(rc);
6279 }
6280
6281#if HC_ARCH_BITS == 64
6282 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6283 {
6284 if ( pVM->hm.s.fAllow64BitGuests
6285 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6286 pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6287 }
6288
6289 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6290 {
6291 if ( pVM->hm.s.fAllow64BitGuests
6292 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6293 {
6294 pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6295 pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6296 pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6297 }
6298 }
6299#endif
6300
6301 if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
6302#if HC_ARCH_BITS == 32
6303 || (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS))
6304#endif
6305 )
6306 {
6307 PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6308 uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6309 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6310 {
6311 switch (pMsr->u32Msr)
6312 {
6313#if HC_ARCH_BITS == 32
6314 case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6315 case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6316 case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6317 case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6318#endif
6319 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6320 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6321 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit */ break;
6322 default:
6323 {
6324 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6325 ASMSetFlags(fEFlags);
6326 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6327 cMsrs));
6328 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6329 }
6330 }
6331 }
6332 }
6333
6334 if (fWhat & CPUMCTX_EXTRN_DR7)
6335 {
6336 if (!pVCpu->hm.s.fUsingHyperDR7)
6337 {
6338 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6339 rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6340 VMXLOCAL_BREAK_RC(rc);
6341 pCtx->dr[7] = u32Val;
6342 }
6343 }
6344
6345 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6346 {
6347 uint32_t u32Shadow;
6348 if (fWhat & CPUMCTX_EXTRN_CR0)
6349 {
6350 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6351 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6352 VMXLOCAL_BREAK_RC(rc);
6353 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32CR0Mask)
6354 | (u32Shadow & pVCpu->hm.s.vmx.u32CR0Mask);
6355 VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6356 CPUMSetGuestCR0(pVCpu, u32Val);
6357 VMMRZCallRing3Enable(pVCpu);
6358 }
6359
6360 if (fWhat & CPUMCTX_EXTRN_CR4)
6361 {
6362 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6363 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6364 VMXLOCAL_BREAK_RC(rc);
6365 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32CR4Mask)
6366 | (u32Shadow & pVCpu->hm.s.vmx.u32CR4Mask);
6367 CPUMSetGuestCR4(pVCpu, u32Val);
6368 }
6369
6370 if (fWhat & CPUMCTX_EXTRN_CR3)
6371 {
6372 /* CR0.PG bit changes are always intercepted, so it's up to date. */
6373 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6374 || ( pVM->hm.s.fNestedPaging
6375 && CPUMIsGuestPagingEnabledEx(pCtx)))
6376 {
6377 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6378 if (pCtx->cr3 != u64Val)
6379 {
6380 CPUMSetGuestCR3(pVCpu, u64Val);
6381 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6382 }
6383
6384 /* If the guest is in PAE mode, sync back the PDPE's into the guest state.
6385 Note: CR4.PAE, CR0.PG, EFER bit changes are always intercepted, so they're up to date. */
6386 if (CPUMIsGuestInPAEModeEx(pCtx))
6387 {
6388 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6389 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6390 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6391 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6392 VMXLOCAL_BREAK_RC(rc);
6393 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6394 }
6395 }
6396 }
6397 }
6398 } while (0);
6399
6400 if (RT_SUCCESS(rc))
6401 {
6402 /* Update fExtrn. */
6403 pCtx->fExtrn &= ~fWhat;
6404
6405 /* If everything has been imported, clear the HM keeper bit. */
6406 if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6407 {
6408 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6409 Assert(!pCtx->fExtrn);
6410 }
6411 }
6412 }
6413 else
6414 AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6415
6416 ASMSetFlags(fEFlags);
6417
6418 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6419
6420 /*
6421 * Honor any pending CR3 updates.
6422 *
6423 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6424 * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6425 * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6426 *
6427 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6428 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6429 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6430 * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6431 *
6432 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6433 */
6434 if (VMMRZCallRing3IsEnabled(pVCpu))
6435 {
6436 VMMR0LogFlushDisable(pVCpu);
6437
6438 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6439 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6440
6441 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6442 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6443
6444 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6445 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6446
6447 VMMR0LogFlushEnable(pVCpu);
6448 }
6449
6450 return VINF_SUCCESS;
6451#undef VMXLOCAL_BREAK_RC
6452}
6453
6454
6455/**
6456 * Saves the guest state from the VMCS into the guest-CPU context.
6457 *
6458 * @returns VBox status code.
6459 * @param pVCpu The cross context virtual CPU structure.
6460 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6461 */
6462VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6463{
6464 return hmR0VmxImportGuestState(pVCpu, fWhat);
6465}
6466
6467
6468/**
6469 * Check per-VM and per-VCPU force flag actions that require us to go back to
6470 * ring-3 for one reason or another.
6471 *
6472 * @returns Strict VBox status code (i.e. informational status codes too)
6473 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6474 * ring-3.
6475 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6476 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6477 * interrupts)
6478 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6479 * all EMTs to be in ring-3.
6480 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6481 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6482 * to the EM loop.
6483 *
6484 * @param pVCpu The cross context virtual CPU structure.
6485 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6486 * out-of-sync. Make sure to update the required fields
6487 * before using them.
6488 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6489 */
6490static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6491{
6492 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6493
6494 /* Pending HM CR3 sync. */
6495 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6496 {
6497 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
6498 int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6499 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6500 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6501 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6502 }
6503
6504 /* Pending HM PAE PDPEs. */
6505 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6506 {
6507 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6508 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6509 }
6510
6511 /*
6512 * Anything pending? Should be more likely than not if we're doing a good job.
6513 */
6514 PVM pVM = pVCpu->CTX_SUFF(pVM);
6515 if ( !fStepping
6516 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6517 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6518 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6519 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6520 return VINF_SUCCESS;
6521
6522 /* Pending PGM C3 sync. */
6523 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6524 {
6525 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
6526 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6527 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6528 if (rcStrict2 != VINF_SUCCESS)
6529 {
6530 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6531 Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6532 return rcStrict2;
6533 }
6534 }
6535
6536 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6537 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6538 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6539 {
6540 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6541 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6542 Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6543 return rc2;
6544 }
6545
6546 /* Pending VM request packets, such as hardware interrupts. */
6547 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6548 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6549 {
6550 Log4Func(("Pending VM request forcing us back to ring-3\n"));
6551 return VINF_EM_PENDING_REQUEST;
6552 }
6553
6554 /* Pending PGM pool flushes. */
6555 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6556 {
6557 Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6558 return VINF_PGM_POOL_FLUSH_PENDING;
6559 }
6560
6561 /* Pending DMA requests. */
6562 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6563 {
6564 Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6565 return VINF_EM_RAW_TO_R3;
6566 }
6567
6568 return VINF_SUCCESS;
6569}
6570
6571
6572/**
6573 * Converts any TRPM trap into a pending HM event. This is typically used when
6574 * entering from ring-3 (not longjmp returns).
6575 *
6576 * @param pVCpu The cross context virtual CPU structure.
6577 */
6578static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6579{
6580 Assert(TRPMHasTrap(pVCpu));
6581 Assert(!pVCpu->hm.s.Event.fPending);
6582
6583 uint8_t uVector;
6584 TRPMEVENT enmTrpmEvent;
6585 RTGCUINT uErrCode;
6586 RTGCUINTPTR GCPtrFaultAddress;
6587 uint8_t cbInstr;
6588
6589 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6590 AssertRC(rc);
6591
6592 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6593 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6594 if (enmTrpmEvent == TRPM_TRAP)
6595 {
6596 switch (uVector)
6597 {
6598 case X86_XCPT_NMI:
6599 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6600 break;
6601
6602 case X86_XCPT_BP:
6603 case X86_XCPT_OF:
6604 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6605 break;
6606
6607 case X86_XCPT_PF:
6608 case X86_XCPT_DF:
6609 case X86_XCPT_TS:
6610 case X86_XCPT_NP:
6611 case X86_XCPT_SS:
6612 case X86_XCPT_GP:
6613 case X86_XCPT_AC:
6614 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6615 RT_FALL_THRU();
6616 default:
6617 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6618 break;
6619 }
6620 }
6621 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6622 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6623 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6624 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6625 else
6626 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6627
6628 rc = TRPMResetTrap(pVCpu);
6629 AssertRC(rc);
6630 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6631 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6632
6633 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6634}
6635
6636
6637/**
6638 * Converts the pending HM event into a TRPM trap.
6639 *
6640 * @param pVCpu The cross context virtual CPU structure.
6641 */
6642static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6643{
6644 Assert(pVCpu->hm.s.Event.fPending);
6645
6646 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6647 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6648 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6649 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6650
6651 /* If a trap was already pending, we did something wrong! */
6652 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6653
6654 TRPMEVENT enmTrapType;
6655 switch (uVectorType)
6656 {
6657 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6658 enmTrapType = TRPM_HARDWARE_INT;
6659 break;
6660
6661 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6662 enmTrapType = TRPM_SOFTWARE_INT;
6663 break;
6664
6665 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6666 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6667 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6668 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6669 enmTrapType = TRPM_TRAP;
6670 break;
6671
6672 default:
6673 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6674 enmTrapType = TRPM_32BIT_HACK;
6675 break;
6676 }
6677
6678 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6679
6680 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6681 AssertRC(rc);
6682
6683 if (fErrorCodeValid)
6684 TRPMSetErrorCode(pVCpu, uErrorCode);
6685
6686 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6687 && uVector == X86_XCPT_PF)
6688 {
6689 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6690 }
6691 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6692 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6693 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6694 {
6695 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6696 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6697 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6698 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6699 }
6700
6701 /* Clear any pending events from the VMCS. */
6702 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
6703 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
6704
6705 /* We're now done converting the pending event. */
6706 pVCpu->hm.s.Event.fPending = false;
6707}
6708
6709
6710/**
6711 * Does the necessary state syncing before returning to ring-3 for any reason
6712 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6713 *
6714 * @returns VBox status code.
6715 * @param pVCpu The cross context virtual CPU structure.
6716 * @param fImportState Whether to import the guest state from the VMCS back
6717 * to the guest-CPU context.
6718 *
6719 * @remarks No-long-jmp zone!!!
6720 */
6721static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
6722{
6723 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6724 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6725
6726 RTCPUID idCpu = RTMpCpuId();
6727 Log4Func(("HostCpuId=%u\n", idCpu));
6728
6729 /*
6730 * !!! IMPORTANT !!!
6731 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
6732 */
6733
6734 /* Save the guest state if necessary. */
6735 if (fImportState)
6736 {
6737 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
6738 AssertRCReturn(rc, rc);
6739 }
6740
6741 /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
6742 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
6743 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6744
6745 /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
6746#ifdef VBOX_STRICT
6747 if (CPUMIsHyperDebugStateActive(pVCpu))
6748 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6749#endif
6750 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6751 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6752 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6753
6754#if HC_ARCH_BITS == 64
6755 /* Restore host-state bits that VT-x only restores partially. */
6756 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6757 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6758 {
6759 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6760 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6761 }
6762 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6763#endif
6764
6765 /* Restore the lazy host MSRs as we're leaving VT-x context. */
6766 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
6767 {
6768 /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
6769 if (!fImportState)
6770 {
6771 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE
6772 | CPUMCTX_EXTRN_SYSCALL_MSRS);
6773 AssertRCReturn(rc, rc);
6774 }
6775 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6776 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
6777 }
6778 else
6779 pVCpu->hm.s.vmx.fLazyMsrs = 0;
6780
6781 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6782 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6783
6784 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6785 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
6786 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
6787 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
6788 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
6789 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6790 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6791 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6792 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6793
6794 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6795
6796 /** @todo This partially defeats the purpose of having preemption hooks.
6797 * The problem is, deregistering the hooks should be moved to a place that
6798 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6799 * context.
6800 */
6801 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6802 {
6803 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6804 AssertRCReturn(rc, rc);
6805
6806 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6807 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6808 }
6809 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6810 NOREF(idCpu);
6811
6812 return VINF_SUCCESS;
6813}
6814
6815
6816/**
6817 * Leaves the VT-x session.
6818 *
6819 * @returns VBox status code.
6820 * @param pVCpu The cross context virtual CPU structure.
6821 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6822 * out-of-sync. Make sure to update the required fields
6823 * before using them.
6824 *
6825 * @remarks No-long-jmp zone!!!
6826 */
6827static int hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6828{
6829 HM_DISABLE_PREEMPT();
6830 HMVMX_ASSERT_CPU_SAFE();
6831 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6832 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6833
6834 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6835 and done this from the VMXR0ThreadCtxCallback(). */
6836 if (!pVCpu->hm.s.fLeaveDone)
6837 {
6838 int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
6839 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
6840 pVCpu->hm.s.fLeaveDone = true;
6841 }
6842 Assert(!pMixedCtx->fExtrn); NOREF(pMixedCtx);
6843
6844 /*
6845 * !!! IMPORTANT !!!
6846 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6847 */
6848
6849 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6850 /** @todo Deregistering here means we need to VMCLEAR always
6851 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6852 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
6853 VMMR0ThreadCtxHookDisable(pVCpu);
6854
6855 /* Leave HM context. This takes care of local init (term). */
6856 int rc = HMR0LeaveCpu(pVCpu);
6857
6858 HM_RESTORE_PREEMPT();
6859 return rc;
6860}
6861
6862
6863/**
6864 * Does the necessary state syncing before doing a longjmp to ring-3.
6865 *
6866 * @returns VBox status code.
6867 * @param pVCpu The cross context virtual CPU structure.
6868 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6869 * out-of-sync. Make sure to update the required fields
6870 * before using them.
6871 *
6872 * @remarks No-long-jmp zone!!!
6873 */
6874DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6875{
6876 return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
6877}
6878
6879
6880/**
6881 * Take necessary actions before going back to ring-3.
6882 *
6883 * An action requires us to go back to ring-3. This function does the necessary
6884 * steps before we can safely return to ring-3. This is not the same as longjmps
6885 * to ring-3, this is voluntary and prepares the guest so it may continue
6886 * executing outside HM (recompiler/IEM).
6887 *
6888 * @returns VBox status code.
6889 * @param pVCpu The cross context virtual CPU structure.
6890 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6891 * out-of-sync. Make sure to update the required fields
6892 * before using them.
6893 * @param rcExit The reason for exiting to ring-3. Can be
6894 * VINF_VMM_UNKNOWN_RING3_CALL.
6895 */
6896static int hmR0VmxExitToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
6897{
6898 Assert(pVCpu);
6899 Assert(pMixedCtx);
6900 HMVMX_ASSERT_PREEMPT_SAFE();
6901
6902 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6903 {
6904 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6905 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6906 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6907 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6908 }
6909
6910 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6911 VMMRZCallRing3Disable(pVCpu);
6912 Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
6913
6914 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6915 if (pVCpu->hm.s.Event.fPending)
6916 {
6917 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6918 Assert(!pVCpu->hm.s.Event.fPending);
6919 }
6920
6921 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
6922 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
6923
6924 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
6925 and if we're injecting an event we should have a TRPM trap pending. */
6926 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6927#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
6928 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6929#endif
6930
6931 /* Save guest state and restore host state bits. */
6932 int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
6933 AssertRCReturn(rc, rc);
6934 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6935 /* Thread-context hooks are unregistered at this point!!! */
6936
6937 /* Sync recompiler state. */
6938 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6939 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6940 | CPUM_CHANGED_LDTR
6941 | CPUM_CHANGED_GDTR
6942 | CPUM_CHANGED_IDTR
6943 | CPUM_CHANGED_TR
6944 | CPUM_CHANGED_HIDDEN_SEL_REGS);
6945 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
6946 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
6947 {
6948 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6949 }
6950
6951 Assert(!pVCpu->hm.s.fClearTrapFlag);
6952
6953 /* Update the exit-to-ring 3 reason. */
6954 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
6955
6956 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6957 if (rcExit != VINF_EM_RAW_INTERRUPT)
6958 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
6959
6960 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6961
6962 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6963 VMMRZCallRing3RemoveNotification(pVCpu);
6964 VMMRZCallRing3Enable(pVCpu);
6965
6966 return rc;
6967}
6968
6969
6970/**
6971 * VMMRZCallRing3() callback wrapper which saves the guest state before we
6972 * longjump to ring-3 and possibly get preempted.
6973 *
6974 * @returns VBox status code.
6975 * @param pVCpu The cross context virtual CPU structure.
6976 * @param enmOperation The operation causing the ring-3 longjump.
6977 * @param pvUser Opaque pointer to the guest-CPU context. The data
6978 * may be out-of-sync. Make sure to update the required
6979 * fields before using them.
6980 */
6981static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6982{
6983 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
6984 {
6985 /*
6986 * !!! IMPORTANT !!!
6987 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
6988 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
6989 */
6990 VMMRZCallRing3RemoveNotification(pVCpu);
6991 VMMRZCallRing3Disable(pVCpu);
6992 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
6993 RTThreadPreemptDisable(&PreemptState);
6994
6995 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
6996 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6997
6998#if HC_ARCH_BITS == 64
6999 /* Restore host-state bits that VT-x only restores partially. */
7000 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7001 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7002 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7003 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7004#endif
7005
7006 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7007 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7008 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7009
7010 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7011 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7012 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7013 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7014 {
7015 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7016 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7017 }
7018
7019 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7020 VMMR0ThreadCtxHookDisable(pVCpu);
7021 HMR0LeaveCpu(pVCpu);
7022 RTThreadPreemptRestore(&PreemptState);
7023 return VINF_SUCCESS;
7024 }
7025
7026 Assert(pVCpu);
7027 Assert(pvUser);
7028 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7029 HMVMX_ASSERT_PREEMPT_SAFE();
7030
7031 VMMRZCallRing3Disable(pVCpu);
7032 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7033
7034 Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7035
7036 int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7037 AssertRCReturn(rc, rc);
7038
7039 VMMRZCallRing3Enable(pVCpu);
7040 return VINF_SUCCESS;
7041}
7042
7043
7044/**
7045 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7046 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7047 *
7048 * @param pVCpu The cross context virtual CPU structure.
7049 */
7050DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7051{
7052 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7053 {
7054 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7055 {
7056 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7057 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7058 AssertRC(rc);
7059 Log4Func(("Setup interrupt-window exiting\n"));
7060 }
7061 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7062}
7063
7064
7065/**
7066 * Clears the interrupt-window exiting control in the VMCS.
7067 *
7068 * @param pVCpu The cross context virtual CPU structure.
7069 */
7070DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7071{
7072 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7073 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7074 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7075 AssertRC(rc);
7076 Log4Func(("Cleared interrupt-window exiting\n"));
7077}
7078
7079
7080/**
7081 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7082 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7083 *
7084 * @param pVCpu The cross context virtual CPU structure.
7085 */
7086DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7087{
7088 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7089 {
7090 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7091 {
7092 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7093 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7094 AssertRC(rc);
7095 Log4Func(("Setup NMI-window exiting\n"));
7096 }
7097 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7098}
7099
7100
7101/**
7102 * Clears the NMI-window exiting control in the VMCS.
7103 *
7104 * @param pVCpu The cross context virtual CPU structure.
7105 */
7106DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7107{
7108 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7109 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7110 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7111 AssertRC(rc);
7112 Log4Func(("Cleared NMI-window exiting\n"));
7113}
7114
7115
7116/**
7117 * Evaluates the event to be delivered to the guest and sets it as the pending
7118 * event.
7119 *
7120 * @returns The VT-x guest-interruptibility state.
7121 * @param pVCpu The cross context virtual CPU structure.
7122 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7123 * out-of-sync. Make sure to update the required fields
7124 * before using them.
7125 */
7126static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7127{
7128 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7129 uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7130 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7131 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7132 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7133
7134 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7135 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7136 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7137 Assert(!TRPMHasTrap(pVCpu));
7138
7139 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7140 APICUpdatePendingInterrupts(pVCpu);
7141
7142 /*
7143 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7144 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7145 */
7146 /** @todo SMI. SMIs take priority over NMIs. */
7147 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7148 {
7149 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7150 if ( !pVCpu->hm.s.Event.fPending
7151 && !fBlockNmi
7152 && !fBlockSti
7153 && !fBlockMovSS)
7154 {
7155 Log4Func(("Pending NMI\n"));
7156 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7157 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7158
7159 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7160 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7161 }
7162 else
7163 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7164 }
7165 /*
7166 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7167 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7168 */
7169 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7170 && !pVCpu->hm.s.fSingleInstruction)
7171 {
7172 Assert(!DBGFIsStepping(pVCpu));
7173 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7174 AssertRCReturn(rc, 0);
7175 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7176 if ( !pVCpu->hm.s.Event.fPending
7177 && !fBlockInt
7178 && !fBlockSti
7179 && !fBlockMovSS)
7180 {
7181 uint8_t u8Interrupt;
7182 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7183 if (RT_SUCCESS(rc))
7184 {
7185 Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7186 uint32_t u32IntInfo = u8Interrupt
7187 | VMX_EXIT_INTERRUPTION_INFO_VALID
7188 | (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7189
7190 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7191 }
7192 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7193 {
7194 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7195 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7196 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7197
7198 /*
7199 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7200 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7201 * need to re-set this force-flag here.
7202 */
7203 }
7204 else
7205 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7206 }
7207 else
7208 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7209 }
7210
7211 return fIntrState;
7212}
7213
7214
7215/**
7216 * Sets a pending-debug exception to be delivered to the guest if the guest is
7217 * single-stepping in the VMCS.
7218 *
7219 * @param pVCpu The cross context virtual CPU structure.
7220 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7221 * out-of-sync. Make sure to update the required fields
7222 * before using them.
7223 */
7224DECLINLINE(int) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7225{
7226 RT_NOREF(pVCpu);
7227 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS)); NOREF(pMixedCtx);
7228 return VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7229}
7230
7231
7232/**
7233 * Injects any pending events into the guest if the guest is in a state to
7234 * receive them.
7235 *
7236 * @returns Strict VBox status code (i.e. informational status codes too).
7237 * @param pVCpu The cross context virtual CPU structure.
7238 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7239 * out-of-sync. Make sure to update the required fields
7240 * before using them.
7241 * @param fIntrState The VT-x guest-interruptibility state.
7242 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7243 * return VINF_EM_DBG_STEPPED if the event was
7244 * dispatched directly.
7245 */
7246static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t fIntrState, bool fStepping)
7247{
7248 HMVMX_ASSERT_PREEMPT_SAFE();
7249 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7250
7251 bool fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7252 bool fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7253
7254 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7255 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7256 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7257 Assert(!TRPMHasTrap(pVCpu));
7258
7259 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7260 if (pVCpu->hm.s.Event.fPending)
7261 {
7262 /*
7263 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7264 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7265 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7266 *
7267 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7268 */
7269 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7270#ifdef VBOX_STRICT
7271 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7272 {
7273 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7274 Assert(!fBlockInt);
7275 Assert(!fBlockSti);
7276 Assert(!fBlockMovSS);
7277 }
7278 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7279 {
7280 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7281 Assert(!fBlockSti);
7282 Assert(!fBlockMovSS);
7283 Assert(!fBlockNmi);
7284 }
7285#endif
7286 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7287 uIntType));
7288 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7289 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7290 &fIntrState);
7291 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7292
7293 /* Update the interruptibility-state as it could have been changed by
7294 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7295 fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7296 fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7297
7298 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7299 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7300 else
7301 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7302 }
7303
7304 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7305 if ( fBlockSti
7306 || fBlockMovSS)
7307 {
7308 if (!pVCpu->hm.s.fSingleInstruction)
7309 {
7310 /*
7311 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7312 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7313 * See Intel spec. 27.3.4 "Saving Non-Register State".
7314 */
7315 Assert(!DBGFIsStepping(pVCpu));
7316 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7317 AssertRCReturn(rc, rc);
7318 if (pMixedCtx->eflags.Bits.u1TF)
7319 {
7320 int rc2 = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
7321 AssertRCReturn(rc2, rc2);
7322 }
7323 }
7324 else if (pMixedCtx->eflags.Bits.u1TF)
7325 {
7326 /*
7327 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7328 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7329 */
7330 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7331 fIntrState = 0;
7332 }
7333 }
7334
7335 /*
7336 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7337 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7338 */
7339 int rc3 = hmR0VmxExportGuestIntrState(pVCpu, fIntrState);
7340 AssertRCReturn(rc3, rc3);
7341
7342 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7343 NOREF(fBlockMovSS); NOREF(fBlockSti);
7344 return rcStrict;
7345}
7346
7347
7348/**
7349 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7350 *
7351 * @param pVCpu The cross context virtual CPU structure.
7352 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7353 * out-of-sync. Make sure to update the required fields
7354 * before using them.
7355 */
7356DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7357{
7358 NOREF(pMixedCtx);
7359 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7360 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7361}
7362
7363
7364/**
7365 * Injects a double-fault (\#DF) exception into the VM.
7366 *
7367 * @returns Strict VBox status code (i.e. informational status codes too).
7368 * @param pVCpu The cross context virtual CPU structure.
7369 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7370 * out-of-sync. Make sure to update the required fields
7371 * before using them.
7372 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7373 * and should return VINF_EM_DBG_STEPPED if the event
7374 * is injected directly (register modified by us, not
7375 * by hardware on VM-entry).
7376 * @param pfIntrState Pointer to the current guest interruptibility-state.
7377 * This interruptibility-state will be updated if
7378 * necessary. This cannot not be NULL.
7379 */
7380DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCCPUMCTX pMixedCtx, bool fStepping, uint32_t *pfIntrState)
7381{
7382 NOREF(pMixedCtx);
7383 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7384 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7385 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7386 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */, fStepping,
7387 pfIntrState);
7388}
7389
7390
7391/**
7392 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7393 *
7394 * @param pVCpu The cross context virtual CPU structure.
7395 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7396 * out-of-sync. Make sure to update the required fields
7397 * before using them.
7398 */
7399DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7400{
7401 NOREF(pMixedCtx);
7402 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7403 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7404 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7405}
7406
7407
7408/**
7409 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7410 *
7411 * @param pVCpu The cross context virtual CPU structure.
7412 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7413 * out-of-sync. Make sure to update the required fields
7414 * before using them.
7415 * @param cbInstr The value of RIP that is to be pushed on the guest
7416 * stack.
7417 */
7418DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7419{
7420 NOREF(pMixedCtx);
7421 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7422 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7423 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7424}
7425
7426
7427/**
7428 * Injects a general-protection (\#GP) fault into the VM.
7429 *
7430 * @returns Strict VBox status code (i.e. informational status codes too).
7431 * @param pVCpu The cross context virtual CPU structure.
7432 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7433 * out-of-sync. Make sure to update the required fields
7434 * before using them.
7435 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7436 * mode, i.e. in real-mode it's not valid).
7437 * @param u32ErrorCode The error code associated with the \#GP.
7438 * @param fStepping Whether we're running in
7439 * hmR0VmxRunGuestCodeStep() and should return
7440 * VINF_EM_DBG_STEPPED if the event is injected
7441 * directly (register modified by us, not by
7442 * hardware on VM-entry).
7443 * @param pfIntrState Pointer to the current guest interruptibility-state.
7444 * This interruptibility-state will be updated if
7445 * necessary. This cannot not be NULL.
7446 */
7447DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7448 bool fStepping, uint32_t *pfIntrState)
7449{
7450 NOREF(pMixedCtx);
7451 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7452 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7453 if (fErrorCodeValid)
7454 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7455 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */, fStepping,
7456 pfIntrState);
7457}
7458
7459
7460#if 0 /* unused */
7461/**
7462 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7463 * VM.
7464 *
7465 * @param pVCpu The cross context virtual CPU structure.
7466 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7467 * out-of-sync. Make sure to update the required fields
7468 * before using them.
7469 * @param u32ErrorCode The error code associated with the \#GP.
7470 */
7471DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7472{
7473 NOREF(pMixedCtx);
7474 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7475 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7476 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7477 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7478}
7479#endif /* unused */
7480
7481
7482/**
7483 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7484 *
7485 * @param pVCpu The cross context virtual CPU structure.
7486 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7487 * out-of-sync. Make sure to update the required fields
7488 * before using them.
7489 * @param uVector The software interrupt vector number.
7490 * @param cbInstr The value of RIP that is to be pushed on the guest
7491 * stack.
7492 */
7493DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7494{
7495 NOREF(pMixedCtx);
7496 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7497 if ( uVector == X86_XCPT_BP
7498 || uVector == X86_XCPT_OF)
7499 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7500 else
7501 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7502 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7503}
7504
7505
7506/**
7507 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7508 * stack.
7509 *
7510 * @returns Strict VBox status code (i.e. informational status codes too).
7511 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7512 * @param pVM The cross context VM structure.
7513 * @param pMixedCtx Pointer to the guest-CPU context.
7514 * @param uValue The value to push to the guest stack.
7515 */
7516DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7517{
7518 /*
7519 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7520 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7521 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7522 */
7523 if (pMixedCtx->sp == 1)
7524 return VINF_EM_RESET;
7525 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7526 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7527 AssertRC(rc);
7528 return rc;
7529}
7530
7531
7532/**
7533 * Injects an event into the guest upon VM-entry by updating the relevant fields
7534 * in the VM-entry area in the VMCS.
7535 *
7536 * @returns Strict VBox status code (i.e. informational status codes too).
7537 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7538 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7539 *
7540 * @param pVCpu The cross context virtual CPU structure.
7541 * @param u64IntInfo The VM-entry interruption-information field.
7542 * @param cbInstr The VM-entry instruction length in bytes (for
7543 * software interrupts, exceptions and privileged
7544 * software exceptions).
7545 * @param u32ErrCode The VM-entry exception error code.
7546 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7547 * @param pfIntrState Pointer to the current guest interruptibility-state.
7548 * This interruptibility-state will be updated if
7549 * necessary. This cannot not be NULL.
7550 * @param fStepping Whether we're running in
7551 * hmR0VmxRunGuestCodeStep() and should return
7552 * VINF_EM_DBG_STEPPED if the event is injected
7553 * directly (register modified by us, not by
7554 * hardware on VM-entry).
7555 */
7556static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7557 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7558{
7559 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7560 AssertMsg(!RT_HI_U32(u64IntInfo), ("%#RX64\n", u64IntInfo));
7561 Assert(pfIntrState);
7562
7563 PCPUMCTX pMixedCtx = &pVCpu->cpum.GstCtx;
7564 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7565 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7566 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7567
7568#ifdef VBOX_STRICT
7569 /*
7570 * Validate the error-code-valid bit for hardware exceptions.
7571 * No error codes for exceptions in real-mode.
7572 *
7573 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7574 */
7575 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7576 && !CPUMIsGuestInRealModeEx(pMixedCtx))
7577 {
7578 switch (uVector)
7579 {
7580 case X86_XCPT_PF:
7581 case X86_XCPT_DF:
7582 case X86_XCPT_TS:
7583 case X86_XCPT_NP:
7584 case X86_XCPT_SS:
7585 case X86_XCPT_GP:
7586 case X86_XCPT_AC:
7587 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7588 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7589 RT_FALL_THRU();
7590 default:
7591 break;
7592 }
7593 }
7594#endif
7595
7596 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7597 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7598 || !(*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7599
7600 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7601
7602 /*
7603 * Hardware interrupts & exceptions cannot be delivered through the software interrupt
7604 * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7605 * interrupt handler in the (real-mode) guest.
7606 *
7607 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7608 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7609 */
7610 if (CPUMIsGuestInRealModeEx(pMixedCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
7611 {
7612 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7613 {
7614 /*
7615 * For unrestricted execution enabled CPUs running real-mode guests, we must not
7616 * set the deliver-error-code bit.
7617 *
7618 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7619 */
7620 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7621 }
7622 else
7623 {
7624 PVM pVM = pVCpu->CTX_SUFF(pVM);
7625 Assert(PDMVmmDevHeapIsEnabled(pVM));
7626 Assert(pVM->hm.s.vmx.pRealModeTSS);
7627
7628 /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7629 int rc2 = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK
7630 | CPUMCTX_EXTRN_TABLE_MASK
7631 | CPUMCTX_EXTRN_RIP
7632 | CPUMCTX_EXTRN_RSP
7633 | CPUMCTX_EXTRN_RFLAGS);
7634 AssertRCReturn(rc2, rc2);
7635
7636 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7637 size_t const cbIdtEntry = sizeof(X86IDTR16);
7638 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7639 {
7640 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7641 if (uVector == X86_XCPT_DF)
7642 return VINF_EM_RESET;
7643
7644 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7645 if (uVector == X86_XCPT_GP)
7646 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, pfIntrState);
7647
7648 /*
7649 * If we're injecting an event with no valid IDT entry, inject a #GP.
7650 * No error codes for exceptions in real-mode.
7651 *
7652 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7653 */
7654 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, fStepping,
7655 pfIntrState);
7656 }
7657
7658 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7659 uint16_t uGuestIp = pMixedCtx->ip;
7660 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7661 {
7662 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7663 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7664 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7665 }
7666 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7667 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7668
7669 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7670 X86IDTR16 IdtEntry;
7671 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7672 rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7673 AssertRCReturn(rc2, rc2);
7674
7675 /* Construct the stack frame for the interrupt/exception handler. */
7676 VBOXSTRICTRC rcStrict;
7677 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7678 if (rcStrict == VINF_SUCCESS)
7679 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7680 if (rcStrict == VINF_SUCCESS)
7681 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7682
7683 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7684 if (rcStrict == VINF_SUCCESS)
7685 {
7686 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7687 pMixedCtx->rip = IdtEntry.offSel;
7688 pMixedCtx->cs.Sel = IdtEntry.uSel;
7689 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7690 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7691 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7692 && uVector == X86_XCPT_PF)
7693 pMixedCtx->cr2 = GCPtrFaultAddress;
7694
7695 /* If any other guest-state bits are changed here, make sure to update
7696 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7697 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS
7698 | HM_CHANGED_GUEST_CR2
7699 | HM_CHANGED_GUEST_RIP
7700 | HM_CHANGED_GUEST_RFLAGS
7701 | HM_CHANGED_GUEST_RSP);
7702
7703 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7704 if (*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7705 {
7706 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7707 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7708 Log4Func(("Clearing inhibition due to STI\n"));
7709 *pfIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7710 }
7711 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7712 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7713
7714 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7715 it, if we are returning to ring-3 before executing guest code. */
7716 pVCpu->hm.s.Event.fPending = false;
7717
7718 /* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
7719 if (fStepping)
7720 rcStrict = VINF_EM_DBG_STEPPED;
7721 }
7722 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7723 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7724 return rcStrict;
7725 }
7726 }
7727
7728 /* Validate. */
7729 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7730 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7731 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7732
7733 /* Inject. */
7734 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7735 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7736 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7737 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7738 AssertRCReturn(rc, rc);
7739
7740 /* Update CR2. */
7741 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7742 && uVector == X86_XCPT_PF)
7743 pMixedCtx->cr2 = GCPtrFaultAddress;
7744
7745 Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7746
7747 return VINF_SUCCESS;
7748}
7749
7750
7751/**
7752 * Clears the interrupt-window exiting control in the VMCS and if necessary
7753 * clears the current event in the VMCS as well.
7754 *
7755 * @returns VBox status code.
7756 * @param pVCpu The cross context virtual CPU structure.
7757 *
7758 * @remarks Use this function only to clear events that have not yet been
7759 * delivered to the guest but are injected in the VMCS!
7760 * @remarks No-long-jump zone!!!
7761 */
7762static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
7763{
7764 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7765 {
7766 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7767 Log4Func(("Cleared interrupt widow\n"));
7768 }
7769
7770 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
7771 {
7772 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
7773 Log4Func(("Cleared interrupt widow\n"));
7774 }
7775}
7776
7777
7778/**
7779 * Enters the VT-x session.
7780 *
7781 * @returns VBox status code.
7782 * @param pVCpu The cross context virtual CPU structure.
7783 * @param pHostCpu Pointer to the global CPU info struct.
7784 */
7785VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu, PHMGLOBALCPUINFO pHostCpu)
7786{
7787 AssertPtr(pVCpu);
7788 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
7789 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7790 RT_NOREF(pHostCpu);
7791
7792 LogFlowFunc(("pVCpu=%p\n", pVCpu));
7793 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7794 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7795
7796#ifdef VBOX_STRICT
7797 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
7798 RTCCUINTREG uHostCR4 = ASMGetCR4();
7799 if (!(uHostCR4 & X86_CR4_VMXE))
7800 {
7801 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7802 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7803 }
7804#endif
7805
7806 /*
7807 * Load the VCPU's VMCS as the current (and active) one.
7808 */
7809 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7810 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7811 if (RT_FAILURE(rc))
7812 return rc;
7813
7814 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7815 pVCpu->hm.s.fLeaveDone = false;
7816 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7817
7818 return VINF_SUCCESS;
7819}
7820
7821
7822/**
7823 * The thread-context callback (only on platforms which support it).
7824 *
7825 * @param enmEvent The thread-context event.
7826 * @param pVCpu The cross context virtual CPU structure.
7827 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7828 * @thread EMT(pVCpu)
7829 */
7830VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7831{
7832 NOREF(fGlobalInit);
7833
7834 switch (enmEvent)
7835 {
7836 case RTTHREADCTXEVENT_OUT:
7837 {
7838 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7839 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7840 VMCPU_ASSERT_EMT(pVCpu);
7841
7842 /* No longjmps (logger flushes, locks) in this fragile context. */
7843 VMMRZCallRing3Disable(pVCpu);
7844 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7845
7846 /*
7847 * Restore host-state (FPU, debug etc.)
7848 */
7849 if (!pVCpu->hm.s.fLeaveDone)
7850 {
7851 /*
7852 * Do -not- import the guest-state here as we might already be in the middle of importing
7853 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
7854 */
7855 hmR0VmxLeave(pVCpu, false /* fImportState */);
7856 pVCpu->hm.s.fLeaveDone = true;
7857 }
7858
7859 /* Leave HM context, takes care of local init (term). */
7860 int rc = HMR0LeaveCpu(pVCpu);
7861 AssertRC(rc); NOREF(rc);
7862
7863 /* Restore longjmp state. */
7864 VMMRZCallRing3Enable(pVCpu);
7865 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
7866 break;
7867 }
7868
7869 case RTTHREADCTXEVENT_IN:
7870 {
7871 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7872 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7873 VMCPU_ASSERT_EMT(pVCpu);
7874
7875 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7876 VMMRZCallRing3Disable(pVCpu);
7877 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7878
7879 /* Initialize the bare minimum state required for HM. This takes care of
7880 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7881 int rc = hmR0EnterCpu(pVCpu);
7882 AssertRC(rc);
7883 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7884 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7885
7886 /* Load the active VMCS as the current one. */
7887 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7888 {
7889 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7890 AssertRC(rc); NOREF(rc);
7891 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7892 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7893 }
7894 pVCpu->hm.s.fLeaveDone = false;
7895
7896 /* Restore longjmp state. */
7897 VMMRZCallRing3Enable(pVCpu);
7898 break;
7899 }
7900
7901 default:
7902 break;
7903 }
7904}
7905
7906
7907/**
7908 * Exports the host state into the VMCS host-state area.
7909 * Sets up the VM-exit MSR-load area.
7910 *
7911 * The CPU state will be loaded from these fields on every successful VM-exit.
7912 *
7913 * @returns VBox status code.
7914 * @param pVCpu The cross context virtual CPU structure.
7915 *
7916 * @remarks No-long-jump zone!!!
7917 */
7918static int hmR0VmxExportHostState(PVMCPU pVCpu)
7919{
7920 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7921
7922 int rc = VINF_SUCCESS;
7923 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
7924 {
7925 rc = hmR0VmxExportHostControlRegs();
7926 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7927
7928 rc = hmR0VmxExportHostSegmentRegs(pVCpu);
7929 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7930
7931 rc = hmR0VmxExportHostMsrs(pVCpu);
7932 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7933
7934 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
7935 }
7936 return rc;
7937}
7938
7939
7940/**
7941 * Saves the host state in the VMCS host-state.
7942 *
7943 * @returns VBox status code.
7944 * @param pVCpu The cross context virtual CPU structure.
7945 *
7946 * @remarks No-long-jump zone!!!
7947 */
7948VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
7949{
7950 AssertPtr(pVCpu);
7951 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7952
7953 /*
7954 * Export the host state here while entering HM context.
7955 * When thread-context hooks are used, we might get preempted and have to re-save the host
7956 * state but most of the time we won't be, so do it here before we disable interrupts.
7957 */
7958 return hmR0VmxExportHostState(pVCpu);
7959}
7960
7961
7962/**
7963 * Exports the guest state into the VMCS guest-state area.
7964 *
7965 * The will typically be done before VM-entry when the guest-CPU state and the
7966 * VMCS state may potentially be out of sync.
7967 *
7968 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
7969 * VM-entry controls.
7970 * Sets up the appropriate VMX non-root function to execute guest code based on
7971 * the guest CPU mode.
7972 *
7973 * @returns VBox strict status code.
7974 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
7975 * without unrestricted guest access and the VMMDev is not presently
7976 * mapped (e.g. EFI32).
7977 *
7978 * @param pVCpu The cross context virtual CPU structure.
7979 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7980 * out-of-sync. Make sure to update the required fields
7981 * before using them.
7982 *
7983 * @remarks No-long-jump zone!!!
7984 */
7985static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
7986{
7987 AssertPtr(pVCpu);
7988 AssertPtr(pMixedCtx);
7989 HMVMX_ASSERT_PREEMPT_SAFE();
7990
7991 LogFlowFunc(("pVCpu=%p\n", pVCpu));
7992
7993 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
7994
7995 /* Determine real-on-v86 mode. */
7996 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7997 if ( !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
7998 && CPUMIsGuestInRealModeEx(pMixedCtx))
7999 {
8000 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8001 }
8002
8003 /*
8004 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8005 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8006 */
8007 int rc = hmR0VmxSelectVMRunHandler(pVCpu, pMixedCtx);
8008 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8009
8010 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8011 rc = hmR0VmxExportGuestEntryCtls(pVCpu, pMixedCtx);
8012 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8013
8014 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8015 rc = hmR0VmxExportGuestExitCtls(pVCpu, pMixedCtx);
8016 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8017
8018 rc = hmR0VmxExportGuestCR0(pVCpu, pMixedCtx);
8019 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8020
8021 VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pMixedCtx);
8022 if (rcStrict == VINF_SUCCESS)
8023 { /* likely */ }
8024 else
8025 {
8026 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8027 return rcStrict;
8028 }
8029
8030 rc = hmR0VmxExportGuestSegmentRegs(pVCpu, pMixedCtx);
8031 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8032
8033 /* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8034 may alter controls if we determine we don't have to swap EFER after all. */
8035 rc = hmR0VmxExportGuestMsrs(pVCpu, pMixedCtx);
8036 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8037
8038 rc = hmR0VmxExportGuestApicTpr(pVCpu);
8039 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8040
8041 /* This needs to be done after hmR0VmxExportGuestCR0() as it may alter intercepted exceptions. */
8042 rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8043 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8044
8045 /* Exporting RFLAGS here is fine, even though RFLAGS.TF might depend on guest debug state which is
8046 not exported here. It is re-evaluated and updated if necessary in hmR0VmxExportSharedState(). */
8047 rc = hmR0VmxExportGuestRip(pVCpu, pMixedCtx);
8048 rc |= hmR0VmxExportGuestRsp(pVCpu, pMixedCtx);
8049 rc |= hmR0VmxExportGuestRflags(pVCpu, pMixedCtx);
8050 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8051
8052 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8053 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8054 | HM_CHANGED_GUEST_CR2
8055 | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8056 | HM_CHANGED_GUEST_X87
8057 | HM_CHANGED_GUEST_SSE_AVX
8058 | HM_CHANGED_GUEST_OTHER_XSAVE
8059 | HM_CHANGED_GUEST_XCRx
8060 | HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8061 | HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8062 | HM_CHANGED_GUEST_TSC_AUX
8063 | HM_CHANGED_GUEST_OTHER_MSRS
8064 | HM_CHANGED_GUEST_HWVIRT
8065 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8066
8067 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8068 return rc;
8069}
8070
8071
8072/**
8073 * Exports the state shared between the host and guest into the VMCS.
8074 *
8075 * @param pVCpu The cross context virtual CPU structure.
8076 * @param pCtx Pointer to the guest-CPU context.
8077 *
8078 * @remarks No-long-jump zone!!!
8079 */
8080static void hmR0VmxExportSharedState(PVMCPU pVCpu, PCPUMCTX pCtx)
8081{
8082 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8083 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8084
8085 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8086 {
8087 int rc = hmR0VmxExportSharedDebugState(pVCpu, pCtx);
8088 AssertRC(rc);
8089 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8090
8091 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8092 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8093 {
8094 rc = hmR0VmxExportGuestRflags(pVCpu, pCtx);
8095 AssertRC(rc);
8096 }
8097 }
8098
8099 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8100 {
8101 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8102 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8103 }
8104
8105 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8106 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8107}
8108
8109
8110/**
8111 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8112 *
8113 * @returns Strict VBox status code (i.e. informational status codes too).
8114 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8115 * without unrestricted guest access and the VMMDev is not presently
8116 * mapped (e.g. EFI32).
8117 *
8118 * @param pVCpu The cross context virtual CPU structure.
8119 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8120 * out-of-sync. Make sure to update the required fields
8121 * before using them.
8122 *
8123 * @remarks No-long-jump zone!!!
8124 */
8125static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
8126{
8127 HMVMX_ASSERT_PREEMPT_SAFE();
8128 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8129 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8130
8131#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8132 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8133#endif
8134
8135 /*
8136 * For many exits it's only RIP that changes and hence try to export it first
8137 * without going through a lot of change flag checks.
8138 */
8139 VBOXSTRICTRC rcStrict;
8140 uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8141 if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8142 {
8143 rcStrict = hmR0VmxExportGuestRip(pVCpu, pMixedCtx);
8144 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8145 { /* likely */}
8146 else
8147 AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8148 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8149 }
8150 else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8151 {
8152 rcStrict = hmR0VmxExportGuestState(pVCpu, pMixedCtx);
8153 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8154 { /* likely */}
8155 else
8156 {
8157 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8158 VBOXSTRICTRC_VAL(rcStrict)));
8159 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8160 return rcStrict;
8161 }
8162 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8163 }
8164 else
8165 rcStrict = VINF_SUCCESS;
8166
8167#ifdef VBOX_STRICT
8168 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8169 fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8170 AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8171 ("fCtxChanged=%#RX64\n", fCtxChanged));
8172#endif
8173 return rcStrict;
8174}
8175
8176
8177/**
8178 * Does the preparations before executing guest code in VT-x.
8179 *
8180 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8181 * recompiler/IEM. We must be cautious what we do here regarding committing
8182 * guest-state information into the VMCS assuming we assuredly execute the
8183 * guest in VT-x mode.
8184 *
8185 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8186 * the common-state (TRPM/forceflags), we must undo those changes so that the
8187 * recompiler/IEM can (and should) use them when it resumes guest execution.
8188 * Otherwise such operations must be done when we can no longer exit to ring-3.
8189 *
8190 * @returns Strict VBox status code (i.e. informational status codes too).
8191 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8192 * have been disabled.
8193 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8194 * double-fault into the guest.
8195 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8196 * dispatched directly.
8197 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8198 *
8199 * @param pVCpu The cross context virtual CPU structure.
8200 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8201 * out-of-sync. Make sure to update the required fields
8202 * before using them.
8203 * @param pVmxTransient Pointer to the VMX transient structure.
8204 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8205 * us ignore some of the reasons for returning to
8206 * ring-3, and return VINF_EM_DBG_STEPPED if event
8207 * dispatching took place.
8208 */
8209static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8210{
8211 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8212
8213#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8214 PGMRZDynMapFlushAutoSet(pVCpu);
8215#endif
8216
8217 /* Check force flag actions that might require us to go back to ring-3. */
8218 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, pMixedCtx, fStepping);
8219 if (rcStrict == VINF_SUCCESS)
8220 { /* FFs doesn't get set all the time. */ }
8221 else
8222 return rcStrict;
8223
8224 /*
8225 * Setup the virtualized-APIC accesses.
8226 *
8227 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8228 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8229 *
8230 * This is the reason we do it here and not in hmR0VmxExportGuestState().
8231 */
8232 PVM pVM = pVCpu->CTX_SUFF(pVM);
8233 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8234 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8235 && PDMHasApic(pVM))
8236 {
8237 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8238 Assert(u64MsrApicBase);
8239 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8240
8241 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8242
8243 /* Unalias any existing mapping. */
8244 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8245 AssertRCReturn(rc, rc);
8246
8247 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8248 Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8249 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8250 AssertRCReturn(rc, rc);
8251
8252 /* Update the per-VCPU cache of the APIC base MSR. */
8253 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8254 }
8255
8256 if (TRPMHasTrap(pVCpu))
8257 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8258 uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8259
8260 /*
8261 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
8262 * needs to be done with longjmps or interrupts + preemption enabled. Event injection might
8263 * also result in triple-faulting the VM.
8264 */
8265 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, fIntrState, fStepping);
8266 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8267 { /* likely */ }
8268 else
8269 {
8270 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8271 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8272 return rcStrict;
8273 }
8274
8275 /*
8276 * No longjmps to ring-3 from this point on!!!
8277 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8278 * This also disables flushing of the R0-logger instance (if any).
8279 */
8280 VMMRZCallRing3Disable(pVCpu);
8281
8282 /*
8283 * Export the guest state bits.
8284 *
8285 * We cannot perform longjmps while loading the guest state because we do not preserve the
8286 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8287 * CPU migration.
8288 *
8289 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8290 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8291 * Hence, loading of the guest state needs to be done -after- injection of events.
8292 */
8293 rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pMixedCtx);
8294 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8295 { /* likely */ }
8296 else
8297 {
8298 VMMRZCallRing3Enable(pVCpu);
8299 return rcStrict;
8300 }
8301
8302 /*
8303 * We disable interrupts so that we don't miss any interrupts that would flag preemption
8304 * (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
8305 * preemption disabled for a while. Since this is purly to aid the
8306 * RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
8307 * disable interrupt on NT.
8308 *
8309 * We need to check for force-flags that could've possible been altered since we last
8310 * checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
8311 * see @bugref{6398}).
8312 *
8313 * We also check a couple of other force-flags as a last opportunity to get the EMT back
8314 * to ring-3 before executing guest code.
8315 */
8316 pVmxTransient->fEFlags = ASMIntDisableFlags();
8317
8318 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8319 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8320 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8321 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8322 {
8323 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8324 {
8325 pVCpu->hm.s.Event.fPending = false;
8326
8327 /*
8328 * We've injected any pending events. This is really the point of no return (to ring-3).
8329 *
8330 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8331 * returns from this function, so don't enable them here.
8332 */
8333 return VINF_SUCCESS;
8334 }
8335
8336 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8337 rcStrict = VINF_EM_RAW_INTERRUPT;
8338 }
8339 else
8340 {
8341 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8342 rcStrict = VINF_EM_RAW_TO_R3;
8343 }
8344
8345 ASMSetFlags(pVmxTransient->fEFlags);
8346 VMMRZCallRing3Enable(pVCpu);
8347
8348 return rcStrict;
8349}
8350
8351
8352/**
8353 * Prepares to run guest code in VT-x and we've committed to doing so. This
8354 * means there is no backing out to ring-3 or anywhere else at this
8355 * point.
8356 *
8357 * @param pVCpu The cross context virtual CPU structure.
8358 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8359 * out-of-sync. Make sure to update the required fields
8360 * before using them.
8361 * @param pVmxTransient Pointer to the VMX transient structure.
8362 *
8363 * @remarks Called with preemption disabled.
8364 * @remarks No-long-jump zone!!!
8365 */
8366static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8367{
8368 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8369 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8370 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8371
8372 /*
8373 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8374 */
8375 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8376 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8377
8378 PVM pVM = pVCpu->CTX_SUFF(pVM);
8379 if (!CPUMIsGuestFPUStateActive(pVCpu))
8380 {
8381 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8382 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8383 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
8384 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8385 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8386 }
8387
8388 /*
8389 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8390 */
8391 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8392 && pVCpu->hm.s.vmx.cMsrs > 0)
8393 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8394
8395 /*
8396 * Re-save the host state bits as we may've been preempted (only happens when
8397 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8398 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8399 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8400 * See @bugref{8432}.
8401 */
8402 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8403 {
8404 int rc = hmR0VmxExportHostState(pVCpu);
8405 AssertRC(rc);
8406 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8407 }
8408 Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8409
8410 /*
8411 * Export the state shared between host and guest (FPU, debug, lazy MSRs).
8412 */
8413 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8414 hmR0VmxExportSharedState(pVCpu, pMixedCtx);
8415 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8416
8417 /* Store status of the shared guest-host state at the time of VM-entry. */
8418#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8419 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8420 {
8421 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8422 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8423 }
8424 else
8425#endif
8426 {
8427 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8428 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8429 }
8430
8431 /*
8432 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8433 */
8434 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8435 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8436
8437 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8438 RTCPUID idCurrentCpu = pCpu->idCpu;
8439 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8440 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8441 {
8442 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8443 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8444 }
8445
8446 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8447 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8448 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8449 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8450
8451 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8452
8453 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8454 to start executing. */
8455
8456 /*
8457 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8458 */
8459 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8460 {
8461 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8462 {
8463 bool fMsrUpdated;
8464 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8465 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8466 &fMsrUpdated);
8467 AssertRC(rc2);
8468 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8469 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8470 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8471 }
8472 else
8473 {
8474 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8475 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8476 }
8477 }
8478
8479 if (pVM->cpum.ro.GuestFeatures.fIbrs)
8480 {
8481 bool fMsrUpdated;
8482 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8483 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8484 &fMsrUpdated);
8485 AssertRC(rc2);
8486 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8487 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8488 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8489 }
8490
8491#ifdef VBOX_STRICT
8492 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8493 hmR0VmxCheckHostEferMsr(pVCpu);
8494 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8495#endif
8496#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8497 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pMixedCtx);
8498 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8499 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8500#endif
8501}
8502
8503
8504/**
8505 * Performs some essential restoration of state after running guest code in
8506 * VT-x.
8507 *
8508 * @param pVCpu The cross context virtual CPU structure.
8509 * @param pVmxTransient Pointer to the VMX transient structure.
8510 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8511 *
8512 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8513 *
8514 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8515 * unconditionally when it is safe to do so.
8516 */
8517static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8518{
8519 uint64_t const uHostTsc = ASMReadTSC();
8520 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8521
8522 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8523 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8524 pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8525 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8526 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8527 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8528
8529 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8530 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TSCOffset);
8531
8532 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
8533 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8534 Assert(!ASMIntAreEnabled());
8535 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8536
8537#if HC_ARCH_BITS == 64
8538 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8539#endif
8540#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8541 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8542 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8543 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8544#else
8545 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8546#endif
8547#ifdef VBOX_STRICT
8548 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8549#endif
8550 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8551
8552 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8553 uint32_t uExitReason;
8554 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8555 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8556 AssertRC(rc);
8557 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8558 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8559
8560 if (rcVMRun == VINF_SUCCESS)
8561 {
8562 /*
8563 * Update the VM-exit history array here even if the VM-entry failed due to:
8564 * - Invalid guest state.
8565 * - MSR loading.
8566 * - Machine-check event.
8567 *
8568 * In any of the above cases we will still have a "valid" VM-exit reason
8569 * despite @a fVMEntryFailed being false.
8570 *
8571 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8572 *
8573 * Note! We don't have CS or RIP at this point. Will probably address that later
8574 * by amending the history entry added here.
8575 */
8576 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8577 UINT64_MAX, uHostTsc);
8578
8579 if (!pVmxTransient->fVMEntryFailed)
8580 {
8581 VMMRZCallRing3Enable(pVCpu);
8582
8583#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8584 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8585 AssertRC(rc);
8586#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8587 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8588 AssertRC(rc);
8589#else
8590 /*
8591 * Import the guest-interruptibility state always as we need it while evaluating
8592 * injecting events on re-entry.
8593 *
8594 * We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8595 * checking for real-mode while exporting the state because all bits that cause
8596 * mode changes wrt CR0 are intercepted.
8597 */
8598 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8599 AssertRC(rc);
8600#endif
8601
8602 /*
8603 * Sync the TPR shadow with our APIC state.
8604 */
8605 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8606 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8607 {
8608 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8609 AssertRC(rc);
8610 ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8611 }
8612
8613 return;
8614 }
8615 }
8616 else
8617 {
8618 Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8619 }
8620
8621 VMMRZCallRing3Enable(pVCpu);
8622}
8623
8624
8625/**
8626 * Runs the guest code using VT-x the normal way.
8627 *
8628 * @returns VBox status code.
8629 * @param pVCpu The cross context virtual CPU structure.
8630 * @param pCtx Pointer to the guest-CPU context.
8631 *
8632 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8633 */
8634static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, PCPUMCTX pCtx)
8635{
8636 VMXTRANSIENT VmxTransient;
8637 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8638 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8639 uint32_t cLoops = 0;
8640
8641 for (;; cLoops++)
8642 {
8643 Assert(!HMR0SuspendPending());
8644 HMVMX_ASSERT_CPU_SAFE();
8645
8646 /* Preparatory work for running guest code, this may force us to return
8647 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8648 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8649 rcStrict = hmR0VmxPreRunGuest(pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8650 if (rcStrict != VINF_SUCCESS)
8651 break;
8652
8653 hmR0VmxPreRunGuestCommitted(pVCpu, pCtx, &VmxTransient);
8654 int rcRun = hmR0VmxRunGuest(pVCpu, pCtx);
8655 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8656
8657 /* Restore any residual host-state and save any bits shared between host
8658 and guest into the guest-CPU state. Re-enables interrupts! */
8659 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8660
8661 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8662 if (RT_SUCCESS(rcRun))
8663 { /* very likely */ }
8664 else
8665 {
8666 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
8667 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, pCtx, &VmxTransient);
8668 return rcRun;
8669 }
8670
8671 /* Profile the VM-exit. */
8672 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8673 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8674 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8675 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
8676 HMVMX_START_EXIT_DISPATCH_PROF();
8677
8678 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8679
8680 /* Handle the VM-exit. */
8681#ifdef HMVMX_USE_FUNCTION_TABLE
8682 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8683#else
8684 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8685#endif
8686 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
8687 if (rcStrict == VINF_SUCCESS)
8688 {
8689 if (cLoops <= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
8690 continue; /* likely */
8691 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8692 rcStrict = VINF_EM_RAW_INTERRUPT;
8693 }
8694 break;
8695 }
8696
8697 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8698 return rcStrict;
8699}
8700
8701
8702
8703/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8704 * probes.
8705 *
8706 * The following few functions and associated structure contains the bloat
8707 * necessary for providing detailed debug events and dtrace probes as well as
8708 * reliable host side single stepping. This works on the principle of
8709 * "subclassing" the normal execution loop and workers. We replace the loop
8710 * method completely and override selected helpers to add necessary adjustments
8711 * to their core operation.
8712 *
8713 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8714 * any performance for debug and analysis features.
8715 *
8716 * @{
8717 */
8718
8719/**
8720 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
8721 * the debug run loop.
8722 */
8723typedef struct VMXRUNDBGSTATE
8724{
8725 /** The RIP we started executing at. This is for detecting that we stepped. */
8726 uint64_t uRipStart;
8727 /** The CS we started executing with. */
8728 uint16_t uCsStart;
8729
8730 /** Whether we've actually modified the 1st execution control field. */
8731 bool fModifiedProcCtls : 1;
8732 /** Whether we've actually modified the 2nd execution control field. */
8733 bool fModifiedProcCtls2 : 1;
8734 /** Whether we've actually modified the exception bitmap. */
8735 bool fModifiedXcptBitmap : 1;
8736
8737 /** We desire the modified the CR0 mask to be cleared. */
8738 bool fClearCr0Mask : 1;
8739 /** We desire the modified the CR4 mask to be cleared. */
8740 bool fClearCr4Mask : 1;
8741 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8742 uint32_t fCpe1Extra;
8743 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
8744 uint32_t fCpe1Unwanted;
8745 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
8746 uint32_t fCpe2Extra;
8747 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
8748 uint32_t bmXcptExtra;
8749 /** The sequence number of the Dtrace provider settings the state was
8750 * configured against. */
8751 uint32_t uDtraceSettingsSeqNo;
8752 /** VM-exits to check (one bit per VM-exit). */
8753 uint32_t bmExitsToCheck[3];
8754
8755 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
8756 uint32_t fProcCtlsInitial;
8757 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
8758 uint32_t fProcCtls2Initial;
8759 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
8760 uint32_t bmXcptInitial;
8761} VMXRUNDBGSTATE;
8762AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
8763typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
8764
8765
8766/**
8767 * Initializes the VMXRUNDBGSTATE structure.
8768 *
8769 * @param pVCpu The cross context virtual CPU structure of the
8770 * calling EMT.
8771 * @param pCtx The CPU register context to go with @a pVCpu.
8772 * @param pDbgState The structure to initialize.
8773 */
8774static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
8775{
8776 pDbgState->uRipStart = pCtx->rip;
8777 pDbgState->uCsStart = pCtx->cs.Sel;
8778
8779 pDbgState->fModifiedProcCtls = false;
8780 pDbgState->fModifiedProcCtls2 = false;
8781 pDbgState->fModifiedXcptBitmap = false;
8782 pDbgState->fClearCr0Mask = false;
8783 pDbgState->fClearCr4Mask = false;
8784 pDbgState->fCpe1Extra = 0;
8785 pDbgState->fCpe1Unwanted = 0;
8786 pDbgState->fCpe2Extra = 0;
8787 pDbgState->bmXcptExtra = 0;
8788 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
8789 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
8790 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
8791}
8792
8793
8794/**
8795 * Updates the VMSC fields with changes requested by @a pDbgState.
8796 *
8797 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
8798 * immediately before executing guest code, i.e. when interrupts are disabled.
8799 * We don't check status codes here as we cannot easily assert or return in the
8800 * latter case.
8801 *
8802 * @param pVCpu The cross context virtual CPU structure.
8803 * @param pDbgState The debug state.
8804 */
8805static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8806{
8807 /*
8808 * Ensure desired flags in VMCS control fields are set.
8809 * (Ignoring write failure here, as we're committed and it's just debug extras.)
8810 *
8811 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
8812 * there should be no stale data in pCtx at this point.
8813 */
8814 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
8815 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
8816 {
8817 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
8818 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
8819 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8820 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
8821 pDbgState->fModifiedProcCtls = true;
8822 }
8823
8824 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
8825 {
8826 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
8827 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
8828 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
8829 pDbgState->fModifiedProcCtls2 = true;
8830 }
8831
8832 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
8833 {
8834 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
8835 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8836 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
8837 pDbgState->fModifiedXcptBitmap = true;
8838 }
8839
8840 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
8841 {
8842 pVCpu->hm.s.vmx.u32CR0Mask = 0;
8843 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
8844 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
8845 }
8846
8847 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
8848 {
8849 pVCpu->hm.s.vmx.u32CR4Mask = 0;
8850 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
8851 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
8852 }
8853}
8854
8855
8856static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
8857{
8858 /*
8859 * Restore VM-exit control settings as we may not reenter this function the
8860 * next time around.
8861 */
8862 /* We reload the initial value, trigger what we can of recalculations the
8863 next time around. From the looks of things, that's all that's required atm. */
8864 if (pDbgState->fModifiedProcCtls)
8865 {
8866 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
8867 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
8868 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
8869 AssertRCReturn(rc2, rc2);
8870 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
8871 }
8872
8873 /* We're currently the only ones messing with this one, so just restore the
8874 cached value and reload the field. */
8875 if ( pDbgState->fModifiedProcCtls2
8876 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
8877 {
8878 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
8879 AssertRCReturn(rc2, rc2);
8880 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
8881 }
8882
8883 /* If we've modified the exception bitmap, we restore it and trigger
8884 reloading and partial recalculation the next time around. */
8885 if (pDbgState->fModifiedXcptBitmap)
8886 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
8887
8888 return rcStrict;
8889}
8890
8891
8892/**
8893 * Configures VM-exit controls for current DBGF and DTrace settings.
8894 *
8895 * This updates @a pDbgState and the VMCS execution control fields to reflect
8896 * the necessary VM-exits demanded by DBGF and DTrace.
8897 *
8898 * @param pVCpu The cross context virtual CPU structure.
8899 * @param pDbgState The debug state.
8900 * @param pVmxTransient Pointer to the VMX transient structure. May update
8901 * fUpdateTscOffsettingAndPreemptTimer.
8902 */
8903static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
8904{
8905 /*
8906 * Take down the dtrace serial number so we can spot changes.
8907 */
8908 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
8909 ASMCompilerBarrier();
8910
8911 /*
8912 * We'll rebuild most of the middle block of data members (holding the
8913 * current settings) as we go along here, so start by clearing it all.
8914 */
8915 pDbgState->bmXcptExtra = 0;
8916 pDbgState->fCpe1Extra = 0;
8917 pDbgState->fCpe1Unwanted = 0;
8918 pDbgState->fCpe2Extra = 0;
8919 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
8920 pDbgState->bmExitsToCheck[i] = 0;
8921
8922 /*
8923 * Software interrupts (INT XXh) - no idea how to trigger these...
8924 */
8925 PVM pVM = pVCpu->CTX_SUFF(pVM);
8926 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
8927 || VBOXVMM_INT_SOFTWARE_ENABLED())
8928 {
8929 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
8930 }
8931
8932 /*
8933 * INT3 breakpoints - triggered by #BP exceptions.
8934 */
8935 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
8936 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
8937
8938 /*
8939 * Exception bitmap and XCPT events+probes.
8940 */
8941 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
8942 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
8943 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
8944
8945 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
8946 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
8947 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
8948 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
8949 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
8950 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
8951 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
8952 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
8953 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
8954 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
8955 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
8956 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
8957 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
8958 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
8959 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
8960 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
8961 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
8962 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
8963
8964 if (pDbgState->bmXcptExtra)
8965 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
8966
8967 /*
8968 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
8969 *
8970 * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
8971 * So, when adding/changing/removing please don't forget to update it.
8972 *
8973 * Some of the macros are picking up local variables to save horizontal space,
8974 * (being able to see it in a table is the lesser evil here).
8975 */
8976#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
8977 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
8978 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
8979#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
8980 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
8981 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
8982 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
8983 } else do { } while (0)
8984#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
8985 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
8986 { \
8987 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
8988 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
8989 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
8990 } else do { } while (0)
8991#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
8992 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
8993 { \
8994 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
8995 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
8996 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
8997 } else do { } while (0)
8998#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
8999 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9000 { \
9001 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9002 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9003 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9004 } else do { } while (0)
9005
9006 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9007 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9008 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9009 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9010 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9011
9012 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9013 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9014 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9015 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9016 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9017 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9018 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9019 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9020 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9021 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9022 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9023 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9024 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9025 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9026 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9027 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9028 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9029 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9030 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9031 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9032 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9033 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9034 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9035 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9036 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9037 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9038 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9039 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9040 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9041 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9042 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9043 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9044 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9045 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9046 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9047 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9048
9049 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9050 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9051 {
9052 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
9053 | CPUMCTX_EXTRN_CR4
9054 | CPUMCTX_EXTRN_APIC_TPR);
9055 AssertRC(rc);
9056
9057#if 0 /** @todo fix me */
9058 pDbgState->fClearCr0Mask = true;
9059 pDbgState->fClearCr4Mask = true;
9060#endif
9061 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9062 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9063 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9064 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9065 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9066 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9067 require clearing here and in the loop if we start using it. */
9068 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9069 }
9070 else
9071 {
9072 if (pDbgState->fClearCr0Mask)
9073 {
9074 pDbgState->fClearCr0Mask = false;
9075 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9076 }
9077 if (pDbgState->fClearCr4Mask)
9078 {
9079 pDbgState->fClearCr4Mask = false;
9080 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9081 }
9082 }
9083 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9084 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9085
9086 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9087 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9088 {
9089 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9090 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9091 }
9092 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9093 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9094
9095 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9096 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9097 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9098 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9099 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9100 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9101 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9102 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9103#if 0 /** @todo too slow, fix handler. */
9104 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9105#endif
9106 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9107
9108 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9109 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9110 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9111 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9112 {
9113 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9114 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9115 }
9116 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9117 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9118 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9119 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9120
9121 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9122 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9123 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9124 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9125 {
9126 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9127 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9128 }
9129 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9130 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9131 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9132 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9133
9134 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9135 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9136 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9137 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9138 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9139 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9140 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9141 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9142 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9143 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9144 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9145 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9146 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9147 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9148 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9149 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9150 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9151 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9152 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9153 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9154 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9155 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9156
9157#undef IS_EITHER_ENABLED
9158#undef SET_ONLY_XBM_IF_EITHER_EN
9159#undef SET_CPE1_XBM_IF_EITHER_EN
9160#undef SET_CPEU_XBM_IF_EITHER_EN
9161#undef SET_CPE2_XBM_IF_EITHER_EN
9162
9163 /*
9164 * Sanitize the control stuff.
9165 */
9166 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9167 if (pDbgState->fCpe2Extra)
9168 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9169 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9170 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9171 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9172 {
9173 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9174 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9175 }
9176
9177 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9178 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9179 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9180 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9181}
9182
9183
9184/**
9185 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9186 * appropriate.
9187 *
9188 * The caller has checked the VM-exit against the
9189 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9190 * already, so we don't have to do that either.
9191 *
9192 * @returns Strict VBox status code (i.e. informational status codes too).
9193 * @param pVCpu The cross context virtual CPU structure.
9194 * @param pMixedCtx Pointer to the guest-CPU context.
9195 * @param pVmxTransient Pointer to the VMX-transient structure.
9196 * @param uExitReason The VM-exit reason.
9197 *
9198 * @remarks The name of this function is displayed by dtrace, so keep it short
9199 * and to the point. No longer than 33 chars long, please.
9200 */
9201static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9202 uint32_t uExitReason)
9203{
9204 /*
9205 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9206 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9207 *
9208 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9209 * does. Must add/change/remove both places. Same ordering, please.
9210 *
9211 * Added/removed events must also be reflected in the next section
9212 * where we dispatch dtrace events.
9213 */
9214 bool fDtrace1 = false;
9215 bool fDtrace2 = false;
9216 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9217 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9218 uint32_t uEventArg = 0;
9219#define SET_EXIT(a_EventSubName) \
9220 do { \
9221 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9222 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9223 } while (0)
9224#define SET_BOTH(a_EventSubName) \
9225 do { \
9226 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9227 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9228 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9229 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9230 } while (0)
9231 switch (uExitReason)
9232 {
9233 case VMX_EXIT_MTF:
9234 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9235
9236 case VMX_EXIT_XCPT_OR_NMI:
9237 {
9238 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9239 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9240 {
9241 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9242 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9243 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9244 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9245 {
9246 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9247 {
9248 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9249 uEventArg = pVmxTransient->uExitIntErrorCode;
9250 }
9251 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9252 switch (enmEvent1)
9253 {
9254 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9255 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9256 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9257 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9258 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9259 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9260 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9261 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9262 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9263 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9264 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9265 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9266 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9267 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9268 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9269 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9270 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9271 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9272 default: break;
9273 }
9274 }
9275 else
9276 AssertFailed();
9277 break;
9278
9279 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9280 uEventArg = idxVector;
9281 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9282 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9283 break;
9284 }
9285 break;
9286 }
9287
9288 case VMX_EXIT_TRIPLE_FAULT:
9289 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9290 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9291 break;
9292 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9293 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9294 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9295 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9296 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9297
9298 /* Instruction specific VM-exits: */
9299 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9300 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9301 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9302 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9303 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9304 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9305 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9306 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9307 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9308 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9309 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9310 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9311 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9312 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9313 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9314 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9315 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9316 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9317 case VMX_EXIT_MOV_CRX:
9318 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9319 if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9320 SET_BOTH(CRX_READ);
9321 else
9322 SET_BOTH(CRX_WRITE);
9323 uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQualification);
9324 break;
9325 case VMX_EXIT_MOV_DRX:
9326 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9327 if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification)
9328 == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9329 SET_BOTH(DRX_READ);
9330 else
9331 SET_BOTH(DRX_WRITE);
9332 uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification);
9333 break;
9334 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9335 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9336 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9337 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9338 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9339 case VMX_EXIT_XDTR_ACCESS:
9340 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9341 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9342 {
9343 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9344 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9345 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9346 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9347 }
9348 break;
9349
9350 case VMX_EXIT_TR_ACCESS:
9351 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9352 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9353 {
9354 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9355 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9356 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9357 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9358 }
9359 break;
9360
9361 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9362 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9363 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9364 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9365 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9366 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9367 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9368 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9369 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9370 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9371 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9372
9373 /* Events that aren't relevant at this point. */
9374 case VMX_EXIT_EXT_INT:
9375 case VMX_EXIT_INT_WINDOW:
9376 case VMX_EXIT_NMI_WINDOW:
9377 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9378 case VMX_EXIT_PREEMPT_TIMER:
9379 case VMX_EXIT_IO_INSTR:
9380 break;
9381
9382 /* Errors and unexpected events. */
9383 case VMX_EXIT_INIT_SIGNAL:
9384 case VMX_EXIT_SIPI:
9385 case VMX_EXIT_IO_SMI:
9386 case VMX_EXIT_SMI:
9387 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9388 case VMX_EXIT_ERR_MSR_LOAD:
9389 case VMX_EXIT_ERR_MACHINE_CHECK:
9390 break;
9391
9392 default:
9393 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9394 break;
9395 }
9396#undef SET_BOTH
9397#undef SET_EXIT
9398
9399 /*
9400 * Dtrace tracepoints go first. We do them here at once so we don't
9401 * have to copy the guest state saving and stuff a few dozen times.
9402 * Down side is that we've got to repeat the switch, though this time
9403 * we use enmEvent since the probes are a subset of what DBGF does.
9404 */
9405 if (fDtrace1 || fDtrace2)
9406 {
9407 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9408 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9409 switch (enmEvent1)
9410 {
9411 /** @todo consider which extra parameters would be helpful for each probe. */
9412 case DBGFEVENT_END: break;
9413 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9414 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9415 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9416 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9417 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9418 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9419 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9420 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9421 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9422 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9423 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9424 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9425 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9426 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9427 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9428 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9429 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9430 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9431 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9432 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9433 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9434 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9435 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9436 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9437 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9438 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9439 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9440 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9441 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9442 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9443 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9444 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9445 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9446 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9447 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9448 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9449 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9450 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9451 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9452 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9453 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9454 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9455 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9456 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9457 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9458 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9459 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9460 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9461 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9462 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9463 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9464 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9465 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9466 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9467 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9468 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9469 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9470 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9471 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9472 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9473 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9474 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9475 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9476 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9477 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9478 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9479 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9480 }
9481 switch (enmEvent2)
9482 {
9483 /** @todo consider which extra parameters would be helpful for each probe. */
9484 case DBGFEVENT_END: break;
9485 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9486 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9487 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9488 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9489 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9490 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9491 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9492 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9493 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9494 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9495 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9496 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9497 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9498 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9499 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9500 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9501 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9502 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9503 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9504 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9505 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9506 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9507 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9508 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9509 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9510 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9511 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9512 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9513 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9514 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9515 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9516 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9517 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9518 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9519 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9520 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9521 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9522 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9523 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9524 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9525 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9526 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9527 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9528 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9529 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9530 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9531 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9532 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9533 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9534 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9535 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9536 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9537 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9538 }
9539 }
9540
9541 /*
9542 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9543 * the DBGF call will do a full check).
9544 *
9545 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9546 * Note! If we have to events, we prioritize the first, i.e. the instruction
9547 * one, in order to avoid event nesting.
9548 */
9549 PVM pVM = pVCpu->CTX_SUFF(pVM);
9550 if ( enmEvent1 != DBGFEVENT_END
9551 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9552 {
9553 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9554 if (rcStrict != VINF_SUCCESS)
9555 return rcStrict;
9556 }
9557 else if ( enmEvent2 != DBGFEVENT_END
9558 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9559 {
9560 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9561 if (rcStrict != VINF_SUCCESS)
9562 return rcStrict;
9563 }
9564
9565 return VINF_SUCCESS;
9566}
9567
9568
9569/**
9570 * Single-stepping VM-exit filtering.
9571 *
9572 * This is preprocessing the VM-exits and deciding whether we've gotten far
9573 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9574 * handling is performed.
9575 *
9576 * @returns Strict VBox status code (i.e. informational status codes too).
9577 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9578 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9579 * out-of-sync. Make sure to update the required
9580 * fields before using them.
9581 * @param pVmxTransient Pointer to the VMX-transient structure.
9582 * @param uExitReason The VM-exit reason.
9583 * @param pDbgState The debug state.
9584 */
9585DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9586 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9587{
9588 /*
9589 * Expensive (saves context) generic dtrace VM-exit probe.
9590 */
9591 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9592 { /* more likely */ }
9593 else
9594 {
9595 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9596 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9597 AssertRC(rc);
9598 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9599 }
9600
9601 /*
9602 * Check for host NMI, just to get that out of the way.
9603 */
9604 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9605 { /* normally likely */ }
9606 else
9607 {
9608 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9609 AssertRCReturn(rc2, rc2);
9610 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9611 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9612 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9613 }
9614
9615 /*
9616 * Check for single stepping event if we're stepping.
9617 */
9618 if (pVCpu->hm.s.fSingleInstruction)
9619 {
9620 switch (uExitReason)
9621 {
9622 case VMX_EXIT_MTF:
9623 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9624
9625 /* Various events: */
9626 case VMX_EXIT_XCPT_OR_NMI:
9627 case VMX_EXIT_EXT_INT:
9628 case VMX_EXIT_TRIPLE_FAULT:
9629 case VMX_EXIT_INT_WINDOW:
9630 case VMX_EXIT_NMI_WINDOW:
9631 case VMX_EXIT_TASK_SWITCH:
9632 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9633 case VMX_EXIT_APIC_ACCESS:
9634 case VMX_EXIT_EPT_VIOLATION:
9635 case VMX_EXIT_EPT_MISCONFIG:
9636 case VMX_EXIT_PREEMPT_TIMER:
9637
9638 /* Instruction specific VM-exits: */
9639 case VMX_EXIT_CPUID:
9640 case VMX_EXIT_GETSEC:
9641 case VMX_EXIT_HLT:
9642 case VMX_EXIT_INVD:
9643 case VMX_EXIT_INVLPG:
9644 case VMX_EXIT_RDPMC:
9645 case VMX_EXIT_RDTSC:
9646 case VMX_EXIT_RSM:
9647 case VMX_EXIT_VMCALL:
9648 case VMX_EXIT_VMCLEAR:
9649 case VMX_EXIT_VMLAUNCH:
9650 case VMX_EXIT_VMPTRLD:
9651 case VMX_EXIT_VMPTRST:
9652 case VMX_EXIT_VMREAD:
9653 case VMX_EXIT_VMRESUME:
9654 case VMX_EXIT_VMWRITE:
9655 case VMX_EXIT_VMXOFF:
9656 case VMX_EXIT_VMXON:
9657 case VMX_EXIT_MOV_CRX:
9658 case VMX_EXIT_MOV_DRX:
9659 case VMX_EXIT_IO_INSTR:
9660 case VMX_EXIT_RDMSR:
9661 case VMX_EXIT_WRMSR:
9662 case VMX_EXIT_MWAIT:
9663 case VMX_EXIT_MONITOR:
9664 case VMX_EXIT_PAUSE:
9665 case VMX_EXIT_XDTR_ACCESS:
9666 case VMX_EXIT_TR_ACCESS:
9667 case VMX_EXIT_INVEPT:
9668 case VMX_EXIT_RDTSCP:
9669 case VMX_EXIT_INVVPID:
9670 case VMX_EXIT_WBINVD:
9671 case VMX_EXIT_XSETBV:
9672 case VMX_EXIT_RDRAND:
9673 case VMX_EXIT_INVPCID:
9674 case VMX_EXIT_VMFUNC:
9675 case VMX_EXIT_RDSEED:
9676 case VMX_EXIT_XSAVES:
9677 case VMX_EXIT_XRSTORS:
9678 {
9679 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
9680 | CPUMCTX_EXTRN_CS);
9681 AssertRCReturn(rc, rc);
9682 if ( pMixedCtx->rip != pDbgState->uRipStart
9683 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
9684 return VINF_EM_DBG_STEPPED;
9685 break;
9686 }
9687
9688 /* Errors and unexpected events: */
9689 case VMX_EXIT_INIT_SIGNAL:
9690 case VMX_EXIT_SIPI:
9691 case VMX_EXIT_IO_SMI:
9692 case VMX_EXIT_SMI:
9693 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9694 case VMX_EXIT_ERR_MSR_LOAD:
9695 case VMX_EXIT_ERR_MACHINE_CHECK:
9696 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9697 break;
9698
9699 default:
9700 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9701 break;
9702 }
9703 }
9704
9705 /*
9706 * Check for debugger event breakpoints and dtrace probes.
9707 */
9708 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9709 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9710 {
9711 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9712 if (rcStrict != VINF_SUCCESS)
9713 return rcStrict;
9714 }
9715
9716 /*
9717 * Normal processing.
9718 */
9719#ifdef HMVMX_USE_FUNCTION_TABLE
9720 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9721#else
9722 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9723#endif
9724}
9725
9726
9727/**
9728 * Single steps guest code using VT-x.
9729 *
9730 * @returns Strict VBox status code (i.e. informational status codes too).
9731 * @param pVCpu The cross context virtual CPU structure.
9732 * @param pCtx Pointer to the guest-CPU context.
9733 *
9734 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
9735 */
9736static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, PCPUMCTX pCtx)
9737{
9738 VMXTRANSIENT VmxTransient;
9739 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9740
9741 /* Set HMCPU indicators. */
9742 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
9743 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
9744 pVCpu->hm.s.fDebugWantRdTscExit = false;
9745 pVCpu->hm.s.fUsingDebugLoop = true;
9746
9747 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
9748 VMXRUNDBGSTATE DbgState;
9749 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
9750 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9751
9752 /*
9753 * The loop.
9754 */
9755 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9756 for (uint32_t cLoops = 0; ; cLoops++)
9757 {
9758 Assert(!HMR0SuspendPending());
9759 HMVMX_ASSERT_CPU_SAFE();
9760 bool fStepping = pVCpu->hm.s.fSingleInstruction;
9761
9762 /*
9763 * Preparatory work for running guest code, this may force us to return
9764 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
9765 */
9766 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9767 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
9768 rcStrict = hmR0VmxPreRunGuest(pVCpu, pCtx, &VmxTransient, fStepping);
9769 if (rcStrict != VINF_SUCCESS)
9770 break;
9771
9772 hmR0VmxPreRunGuestCommitted(pVCpu, pCtx, &VmxTransient);
9773 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
9774
9775 /*
9776 * Now we can run the guest code.
9777 */
9778 int rcRun = hmR0VmxRunGuest(pVCpu, pCtx);
9779
9780 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9781
9782 /*
9783 * Restore any residual host-state and save any bits shared between host
9784 * and guest into the guest-CPU state. Re-enables interrupts!
9785 */
9786 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
9787
9788 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9789 if (RT_SUCCESS(rcRun))
9790 { /* very likely */ }
9791 else
9792 {
9793 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
9794 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, pCtx, &VmxTransient);
9795 return rcRun;
9796 }
9797
9798 /* Profile the VM-exit. */
9799 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9800 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9801 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9802 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
9803 HMVMX_START_EXIT_DISPATCH_PROF();
9804
9805 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9806
9807 /*
9808 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
9809 */
9810 rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
9811 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
9812 if (rcStrict != VINF_SUCCESS)
9813 break;
9814 if (cLoops > pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
9815 {
9816 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9817 rcStrict = VINF_EM_RAW_INTERRUPT;
9818 break;
9819 }
9820
9821 /*
9822 * Stepping: Did the RIP change, if so, consider it a single step.
9823 * Otherwise, make sure one of the TFs gets set.
9824 */
9825 if (fStepping)
9826 {
9827 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
9828 | CPUMCTX_EXTRN_CS);
9829 AssertRC(rc);
9830 if ( pCtx->rip != DbgState.uRipStart
9831 || pCtx->cs.Sel != DbgState.uCsStart)
9832 {
9833 rcStrict = VINF_EM_DBG_STEPPED;
9834 break;
9835 }
9836 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
9837 }
9838
9839 /*
9840 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
9841 */
9842 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
9843 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9844 }
9845
9846 /*
9847 * Clear the X86_EFL_TF if necessary.
9848 */
9849 if (pVCpu->hm.s.fClearTrapFlag)
9850 {
9851 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
9852 AssertRC(rc);
9853 pVCpu->hm.s.fClearTrapFlag = false;
9854 pCtx->eflags.Bits.u1TF = 0;
9855 }
9856 /** @todo there seems to be issues with the resume flag when the monitor trap
9857 * flag is pending without being used. Seen early in bios init when
9858 * accessing APIC page in protected mode. */
9859
9860 /*
9861 * Restore VM-exit control settings as we may not reenter this function the
9862 * next time around.
9863 */
9864 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
9865
9866 /* Restore HMCPU indicators. */
9867 pVCpu->hm.s.fUsingDebugLoop = false;
9868 pVCpu->hm.s.fDebugWantRdTscExit = false;
9869 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
9870
9871 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9872 return rcStrict;
9873}
9874
9875
9876/** @} */
9877
9878
9879/**
9880 * Checks if any expensive dtrace probes are enabled and we should go to the
9881 * debug loop.
9882 *
9883 * @returns true if we should use debug loop, false if not.
9884 */
9885static bool hmR0VmxAnyExpensiveProbesEnabled(void)
9886{
9887 /* It's probably faster to OR the raw 32-bit counter variables together.
9888 Since the variables are in an array and the probes are next to one
9889 another (more or less), we have good locality. So, better read
9890 eight-nine cache lines ever time and only have one conditional, than
9891 128+ conditionals, right? */
9892 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
9893 | VBOXVMM_XCPT_DE_ENABLED_RAW()
9894 | VBOXVMM_XCPT_DB_ENABLED_RAW()
9895 | VBOXVMM_XCPT_BP_ENABLED_RAW()
9896 | VBOXVMM_XCPT_OF_ENABLED_RAW()
9897 | VBOXVMM_XCPT_BR_ENABLED_RAW()
9898 | VBOXVMM_XCPT_UD_ENABLED_RAW()
9899 | VBOXVMM_XCPT_NM_ENABLED_RAW()
9900 | VBOXVMM_XCPT_DF_ENABLED_RAW()
9901 | VBOXVMM_XCPT_TS_ENABLED_RAW()
9902 | VBOXVMM_XCPT_NP_ENABLED_RAW()
9903 | VBOXVMM_XCPT_SS_ENABLED_RAW()
9904 | VBOXVMM_XCPT_GP_ENABLED_RAW()
9905 | VBOXVMM_XCPT_PF_ENABLED_RAW()
9906 | VBOXVMM_XCPT_MF_ENABLED_RAW()
9907 | VBOXVMM_XCPT_AC_ENABLED_RAW()
9908 | VBOXVMM_XCPT_XF_ENABLED_RAW()
9909 | VBOXVMM_XCPT_VE_ENABLED_RAW()
9910 | VBOXVMM_XCPT_SX_ENABLED_RAW()
9911 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
9912 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
9913 ) != 0
9914 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
9915 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
9916 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
9917 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
9918 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
9919 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
9920 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
9921 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
9922 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
9923 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
9924 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
9925 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
9926 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
9927 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
9928 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
9929 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
9930 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
9931 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
9932 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
9933 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
9934 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
9935 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
9936 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
9937 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
9938 | VBOXVMM_INSTR_STR_ENABLED_RAW()
9939 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
9940 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
9941 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
9942 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
9943 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
9944 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
9945 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
9946 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
9947 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
9948 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
9949 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
9950 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
9951 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
9952 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
9953 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
9954 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
9955 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
9956 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
9957 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
9958 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
9959 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
9960 ) != 0
9961 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
9962 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
9963 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
9964 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
9965 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
9966 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
9967 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
9968 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
9969 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
9970 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
9971 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
9972 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
9973 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
9974 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
9975 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
9976 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
9977 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
9978 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
9979 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
9980 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
9981 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
9982 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
9983 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
9984 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
9985 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
9986 | VBOXVMM_EXIT_STR_ENABLED_RAW()
9987 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
9988 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
9989 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
9990 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
9991 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
9992 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
9993 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
9994 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
9995 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
9996 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
9997 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
9998 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
9999 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10000 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10001 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10002 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10003 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10004 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10005 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10006 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10007 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10008 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10009 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10010 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10011 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10012 ) != 0;
10013}
10014
10015
10016/**
10017 * Runs the guest code using VT-x.
10018 *
10019 * @returns Strict VBox status code (i.e. informational status codes too).
10020 * @param pVCpu The cross context virtual CPU structure.
10021 * @param pCtx Pointer to the guest-CPU context.
10022 */
10023VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu, PCPUMCTX pCtx)
10024{
10025 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10026 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10027 HMVMX_ASSERT_PREEMPT_SAFE();
10028
10029 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10030
10031 VBOXSTRICTRC rcStrict;
10032 if ( !pVCpu->hm.s.fUseDebugLoop
10033 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10034 && !DBGFIsStepping(pVCpu)
10035 && !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
10036 rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, pCtx);
10037 else
10038 rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, pCtx);
10039
10040 if (rcStrict == VERR_EM_INTERPRETER)
10041 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10042 else if (rcStrict == VINF_EM_RESET)
10043 rcStrict = VINF_EM_TRIPLE_FAULT;
10044
10045 int rc2 = hmR0VmxExitToRing3(pVCpu, pCtx, rcStrict);
10046 if (RT_FAILURE(rc2))
10047 {
10048 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10049 rcStrict = rc2;
10050 }
10051 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10052 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10053 return rcStrict;
10054}
10055
10056
10057#ifndef HMVMX_USE_FUNCTION_TABLE
10058DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10059{
10060#ifdef DEBUG_ramshankar
10061#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10062 do { \
10063 if (a_fSave != 0) \
10064 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10065 VBOXSTRICTRC rcStrict = a_CallExpr; \
10066 if (a_fSave != 0) \
10067 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10068 return rcStrict; \
10069 } while (0)
10070#else
10071# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10072#endif
10073 switch (rcReason)
10074 {
10075 case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10076 case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10077 case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10078 case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10079 case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10080 case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10081 case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10082 case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10083 case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10084 case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10085 case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10086 case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10087 case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10088 case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10089 case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10090 case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10091 case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10092 case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10093 case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10094 case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10095 case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10096 case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10097 case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10098 case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10099 case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10100 case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10101 case VMX_EXIT_XDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10102 case VMX_EXIT_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10103 case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10104 case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10105 case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10106 case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10107 case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10108 case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10109
10110 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10111 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10112 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10113 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10114 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10115 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10116 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10117 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10118 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10119
10120 case VMX_EXIT_VMCLEAR:
10121 case VMX_EXIT_VMLAUNCH:
10122 case VMX_EXIT_VMPTRLD:
10123 case VMX_EXIT_VMPTRST:
10124 case VMX_EXIT_VMREAD:
10125 case VMX_EXIT_VMRESUME:
10126 case VMX_EXIT_VMWRITE:
10127 case VMX_EXIT_VMXOFF:
10128 case VMX_EXIT_VMXON:
10129 case VMX_EXIT_INVEPT:
10130 case VMX_EXIT_INVVPID:
10131 case VMX_EXIT_VMFUNC:
10132 case VMX_EXIT_XSAVES:
10133 case VMX_EXIT_XRSTORS:
10134 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10135
10136 case VMX_EXIT_ENCLS:
10137 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10138 case VMX_EXIT_PML_FULL:
10139 default:
10140 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10141 }
10142#undef VMEXIT_CALL_RET
10143}
10144#endif /* !HMVMX_USE_FUNCTION_TABLE */
10145
10146
10147#ifdef VBOX_STRICT
10148/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10149# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10150 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10151
10152# define HMVMX_ASSERT_PREEMPT_CPUID() \
10153 do { \
10154 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10155 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10156 } while (0)
10157
10158# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10159 do { \
10160 AssertPtr(pVCpu); \
10161 AssertPtr(pMixedCtx); \
10162 AssertPtr(pVmxTransient); \
10163 Assert(pVmxTransient->fVMEntryFailed == false); \
10164 Assert(ASMIntAreEnabled()); \
10165 HMVMX_ASSERT_PREEMPT_SAFE(); \
10166 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10167 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10168 HMVMX_ASSERT_PREEMPT_SAFE(); \
10169 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10170 HMVMX_ASSERT_PREEMPT_CPUID(); \
10171 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10172 } while (0)
10173
10174# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10175 do { \
10176 Log4Func(("\n")); \
10177 } while (0)
10178#else /* nonstrict builds: */
10179# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10180 do { \
10181 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10182 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10183 } while (0)
10184# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10185#endif
10186
10187
10188/**
10189 * Advances the guest RIP by the specified number of bytes.
10190 *
10191 * @param pVCpu The cross context virtual CPU structure.
10192 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10193 * out-of-sync. Make sure to update the required fields
10194 * before using them.
10195 * @param cbInstr Number of bytes to advance the RIP by.
10196 *
10197 * @remarks No-long-jump zone!!!
10198 */
10199DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10200{
10201 /* Advance the RIP. */
10202 pMixedCtx->rip += cbInstr;
10203 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10204
10205 /* Update interrupt inhibition. */
10206 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10207 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10208 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10209}
10210
10211
10212/**
10213 * Advances the guest RIP after reading it from the VMCS.
10214 *
10215 * @returns VBox status code, no informational status codes.
10216 * @param pVCpu The cross context virtual CPU structure.
10217 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10218 * out-of-sync. Make sure to update the required fields
10219 * before using them.
10220 * @param pVmxTransient Pointer to the VMX transient structure.
10221 *
10222 * @remarks No-long-jump zone!!!
10223 */
10224static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10225{
10226 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10227 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
10228 | CPUMCTX_EXTRN_RFLAGS);
10229 AssertRCReturn(rc, rc);
10230
10231 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10232
10233 /*
10234 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10235 * pending debug exception field as it takes care of priority of events.
10236 *
10237 * See Intel spec. 32.2.1 "Debug Exceptions".
10238 */
10239 if ( !pVCpu->hm.s.fSingleInstruction
10240 && pMixedCtx->eflags.Bits.u1TF)
10241 {
10242 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
10243 AssertRCReturn(rc, rc);
10244 }
10245
10246 return VINF_SUCCESS;
10247}
10248
10249
10250/**
10251 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10252 * and update error record fields accordingly.
10253 *
10254 * @return VMX_IGS_* return codes.
10255 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10256 * wrong with the guest state.
10257 *
10258 * @param pVCpu The cross context virtual CPU structure.
10259 * @param pCtx Pointer to the guest-CPU state.
10260 *
10261 * @remarks This function assumes our cache of the VMCS controls
10262 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10263 */
10264static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCPUMCTX pCtx)
10265{
10266#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10267#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10268 uError = (err); \
10269 break; \
10270 } else do { } while (0)
10271
10272 int rc;
10273 PVM pVM = pVCpu->CTX_SUFF(pVM);
10274 uint32_t uError = VMX_IGS_ERROR;
10275 uint32_t u32Val;
10276 bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10277
10278 do
10279 {
10280 /*
10281 * CR0.
10282 */
10283 uint32_t fSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10284 uint32_t fZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10285 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10286 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10287 if (fUnrestrictedGuest)
10288 fSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
10289
10290 uint32_t uGuestCR0;
10291 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uGuestCR0);
10292 AssertRCBreak(rc);
10293 HMVMX_CHECK_BREAK((uGuestCR0 & fSetCR0) == fSetCR0, VMX_IGS_CR0_FIXED1);
10294 HMVMX_CHECK_BREAK(!(uGuestCR0 & ~fZapCR0), VMX_IGS_CR0_FIXED0);
10295 if ( !fUnrestrictedGuest
10296 && (uGuestCR0 & X86_CR0_PG)
10297 && !(uGuestCR0 & X86_CR0_PE))
10298 {
10299 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10300 }
10301
10302 /*
10303 * CR4.
10304 */
10305 uint64_t fSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10306 uint64_t fZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10307
10308 uint32_t uGuestCR4;
10309 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uGuestCR4);
10310 AssertRCBreak(rc);
10311 HMVMX_CHECK_BREAK((uGuestCR4 & fSetCR4) == fSetCR4, VMX_IGS_CR4_FIXED1);
10312 HMVMX_CHECK_BREAK(!(uGuestCR4 & ~fZapCR4), VMX_IGS_CR4_FIXED0);
10313
10314 /*
10315 * IA32_DEBUGCTL MSR.
10316 */
10317 uint64_t u64Val;
10318 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10319 AssertRCBreak(rc);
10320 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10321 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10322 {
10323 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10324 }
10325 uint64_t u64DebugCtlMsr = u64Val;
10326
10327#ifdef VBOX_STRICT
10328 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10329 AssertRCBreak(rc);
10330 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10331#endif
10332 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10333
10334 /*
10335 * RIP and RFLAGS.
10336 */
10337 uint32_t u32Eflags;
10338#if HC_ARCH_BITS == 64
10339 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10340 AssertRCBreak(rc);
10341 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10342 if ( !fLongModeGuest
10343 || !pCtx->cs.Attr.n.u1Long)
10344 {
10345 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10346 }
10347 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10348 * must be identical if the "IA-32e mode guest" VM-entry
10349 * control is 1 and CS.L is 1. No check applies if the
10350 * CPU supports 64 linear-address bits. */
10351
10352 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10353 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10354 AssertRCBreak(rc);
10355 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10356 VMX_IGS_RFLAGS_RESERVED);
10357 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10358 u32Eflags = u64Val;
10359#else
10360 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10361 AssertRCBreak(rc);
10362 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10363 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10364#endif
10365
10366 if ( fLongModeGuest
10367 || ( fUnrestrictedGuest
10368 && !(uGuestCR0 & X86_CR0_PE)))
10369 {
10370 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10371 }
10372
10373 uint32_t u32EntryInfo;
10374 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10375 AssertRCBreak(rc);
10376 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10377 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10378 {
10379 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10380 }
10381
10382 /*
10383 * 64-bit checks.
10384 */
10385#if HC_ARCH_BITS == 64
10386 if (fLongModeGuest)
10387 {
10388 HMVMX_CHECK_BREAK(uGuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10389 HMVMX_CHECK_BREAK(uGuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10390 }
10391
10392 if ( !fLongModeGuest
10393 && (uGuestCR4 & X86_CR4_PCIDE))
10394 {
10395 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10396 }
10397
10398 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10399 * 51:32 beyond the processor's physical-address width are 0. */
10400
10401 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10402 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10403 {
10404 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10405 }
10406
10407 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10408 AssertRCBreak(rc);
10409 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10410
10411 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10412 AssertRCBreak(rc);
10413 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10414#endif
10415
10416 /*
10417 * PERF_GLOBAL MSR.
10418 */
10419 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10420 {
10421 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10422 AssertRCBreak(rc);
10423 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10424 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10425 }
10426
10427 /*
10428 * PAT MSR.
10429 */
10430 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10431 {
10432 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10433 AssertRCBreak(rc);
10434 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10435 for (unsigned i = 0; i < 8; i++)
10436 {
10437 uint8_t u8Val = (u64Val & 0xff);
10438 if ( u8Val != 0 /* UC */
10439 && u8Val != 1 /* WC */
10440 && u8Val != 4 /* WT */
10441 && u8Val != 5 /* WP */
10442 && u8Val != 6 /* WB */
10443 && u8Val != 7 /* UC- */)
10444 {
10445 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10446 }
10447 u64Val >>= 8;
10448 }
10449 }
10450
10451 /*
10452 * EFER MSR.
10453 */
10454 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10455 {
10456 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10457 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10458 AssertRCBreak(rc);
10459 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10460 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10461 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10462 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10463 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10464 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10465 || !(uGuestCR0 & X86_CR0_PG)
10466 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10467 VMX_IGS_EFER_LMA_LME_MISMATCH);
10468 }
10469
10470 /*
10471 * Segment registers.
10472 */
10473 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10474 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10475 if (!(u32Eflags & X86_EFL_VM))
10476 {
10477 /* CS */
10478 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10479 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10480 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10481 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10482 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10483 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10484 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10485 /* CS cannot be loaded with NULL in protected mode. */
10486 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10487 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10488 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10489 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10490 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10491 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10492 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10493 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10494 else
10495 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10496
10497 /* SS */
10498 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10499 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10500 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10501 if ( !(pCtx->cr0 & X86_CR0_PE)
10502 || pCtx->cs.Attr.n.u4Type == 3)
10503 {
10504 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10505 }
10506 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10507 {
10508 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10509 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10510 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10511 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10512 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10513 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10514 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10515 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10516 }
10517
10518 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
10519 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10520 {
10521 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10522 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10523 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10524 || pCtx->ds.Attr.n.u4Type > 11
10525 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10526 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10527 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10528 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10529 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10530 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10531 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10532 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10533 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10534 }
10535 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10536 {
10537 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10538 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10539 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10540 || pCtx->es.Attr.n.u4Type > 11
10541 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10542 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10543 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10544 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10545 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10546 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10547 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10548 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10549 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10550 }
10551 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10552 {
10553 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10554 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10555 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10556 || pCtx->fs.Attr.n.u4Type > 11
10557 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10558 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10559 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10560 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10561 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10562 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10563 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10564 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10565 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10566 }
10567 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10568 {
10569 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10570 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10571 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10572 || pCtx->gs.Attr.n.u4Type > 11
10573 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10574 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10575 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10576 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10577 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10578 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10579 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10580 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10581 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10582 }
10583 /* 64-bit capable CPUs. */
10584#if HC_ARCH_BITS == 64
10585 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10586 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10587 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10588 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10589 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10590 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10591 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10592 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10593 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10594 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10595 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10596#endif
10597 }
10598 else
10599 {
10600 /* V86 mode checks. */
10601 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10602 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10603 {
10604 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10605 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10606 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10607 }
10608 else
10609 {
10610 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10611 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10612 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10613 }
10614
10615 /* CS */
10616 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10617 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10618 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10619 /* SS */
10620 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10621 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10622 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10623 /* DS */
10624 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10625 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10626 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10627 /* ES */
10628 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10629 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10630 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10631 /* FS */
10632 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10633 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10634 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10635 /* GS */
10636 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10637 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10638 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10639 /* 64-bit capable CPUs. */
10640#if HC_ARCH_BITS == 64
10641 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10642 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10643 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10644 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10645 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10646 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10647 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10648 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10649 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10650 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10651 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10652#endif
10653 }
10654
10655 /*
10656 * TR.
10657 */
10658 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10659 /* 64-bit capable CPUs. */
10660#if HC_ARCH_BITS == 64
10661 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10662#endif
10663 if (fLongModeGuest)
10664 {
10665 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10666 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10667 }
10668 else
10669 {
10670 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10671 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10672 VMX_IGS_TR_ATTR_TYPE_INVALID);
10673 }
10674 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10675 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10676 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10677 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10678 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10679 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10680 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10681 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10682
10683 /*
10684 * GDTR and IDTR.
10685 */
10686#if HC_ARCH_BITS == 64
10687 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10688 AssertRCBreak(rc);
10689 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10690
10691 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10692 AssertRCBreak(rc);
10693 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10694#endif
10695
10696 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10697 AssertRCBreak(rc);
10698 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10699
10700 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10701 AssertRCBreak(rc);
10702 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10703
10704 /*
10705 * Guest Non-Register State.
10706 */
10707 /* Activity State. */
10708 uint32_t u32ActivityState;
10709 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10710 AssertRCBreak(rc);
10711 HMVMX_CHECK_BREAK( !u32ActivityState
10712 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
10713 VMX_IGS_ACTIVITY_STATE_INVALID);
10714 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10715 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10716 uint32_t u32IntrState;
10717 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
10718 AssertRCBreak(rc);
10719 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
10720 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10721 {
10722 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10723 }
10724
10725 /** @todo Activity state and injecting interrupts. Left as a todo since we
10726 * currently don't use activity states but ACTIVE. */
10727
10728 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10729 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10730
10731 /* Guest interruptibility-state. */
10732 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10733 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10734 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
10735 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10736 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10737 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10738 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10739 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10740 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10741 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
10742 {
10743 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10744 {
10745 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10746 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10747 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10748 }
10749 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10750 {
10751 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10752 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10753 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10754 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10755 }
10756 }
10757 /** @todo Assumes the processor is not in SMM. */
10758 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10759 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10760 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10761 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10762 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10763 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
10764 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10765 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10766 {
10767 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
10768 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10769 }
10770
10771 /* Pending debug exceptions. */
10772#if HC_ARCH_BITS == 64
10773 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
10774 AssertRCBreak(rc);
10775 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10776 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10777 u32Val = u64Val; /* For pending debug exceptions checks below. */
10778#else
10779 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
10780 AssertRCBreak(rc);
10781 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10782 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10783#endif
10784
10785 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10786 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
10787 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10788 {
10789 if ( (u32Eflags & X86_EFL_TF)
10790 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10791 {
10792 /* Bit 14 is PendingDebug.BS. */
10793 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10794 }
10795 if ( !(u32Eflags & X86_EFL_TF)
10796 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10797 {
10798 /* Bit 14 is PendingDebug.BS. */
10799 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10800 }
10801 }
10802
10803 /* VMCS link pointer. */
10804 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10805 AssertRCBreak(rc);
10806 if (u64Val != UINT64_C(0xffffffffffffffff))
10807 {
10808 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10809 /** @todo Bits beyond the processor's physical-address width MBZ. */
10810 /** @todo 32-bit located in memory referenced by value of this field (as a
10811 * physical address) must contain the processor's VMCS revision ID. */
10812 /** @todo SMM checks. */
10813 }
10814
10815 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10816 * not using Nested Paging? */
10817 if ( pVM->hm.s.fNestedPaging
10818 && !fLongModeGuest
10819 && CPUMIsGuestInPAEModeEx(pCtx))
10820 {
10821 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10822 AssertRCBreak(rc);
10823 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10824
10825 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10826 AssertRCBreak(rc);
10827 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10828
10829 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10830 AssertRCBreak(rc);
10831 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10832
10833 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10834 AssertRCBreak(rc);
10835 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10836 }
10837
10838 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10839 if (uError == VMX_IGS_ERROR)
10840 uError = VMX_IGS_REASON_NOT_FOUND;
10841 } while (0);
10842
10843 pVCpu->hm.s.u32HMError = uError;
10844 return uError;
10845
10846#undef HMVMX_ERROR_BREAK
10847#undef HMVMX_CHECK_BREAK
10848}
10849
10850/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10851/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
10852/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10853
10854/** @name VM-exit handlers.
10855 * @{
10856 */
10857
10858/**
10859 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
10860 */
10861HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10862{
10863 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10864 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
10865 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
10866 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
10867 return VINF_SUCCESS;
10868 return VINF_EM_RAW_INTERRUPT;
10869}
10870
10871
10872/**
10873 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
10874 */
10875HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10876{
10877 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10878 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
10879
10880 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10881 AssertRCReturn(rc, rc);
10882
10883 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10884 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
10885 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
10886 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
10887
10888 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10889 {
10890 /*
10891 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
10892 * injected it ourselves and anything we inject is not going to cause a VM-exit directly
10893 * for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
10894 *
10895 * [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
10896 * [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
10897 */
10898 VMXDispatchHostNmi();
10899 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
10900 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10901 return VINF_SUCCESS;
10902 }
10903
10904 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10905 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10906 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
10907 { /* likely */ }
10908 else
10909 {
10910 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
10911 rcStrictRc1 = VINF_SUCCESS;
10912 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10913 return rcStrictRc1;
10914 }
10915
10916 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
10917 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
10918 switch (uIntType)
10919 {
10920 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
10921 Assert(uVector == X86_XCPT_DB);
10922 RT_FALL_THRU();
10923 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
10924 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
10925 RT_FALL_THRU();
10926 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
10927 {
10928 /*
10929 * If there's any exception caused as a result of event injection, the resulting
10930 * secondary/final execption will be pending, we shall continue guest execution
10931 * after injecting the event. The page-fault case is complicated and we manually
10932 * handle any currently pending event in hmR0VmxExitXcptPF.
10933 */
10934 if (!pVCpu->hm.s.Event.fPending)
10935 { /* likely */ }
10936 else if (uVector != X86_XCPT_PF)
10937 {
10938 rc = VINF_SUCCESS;
10939 break;
10940 }
10941
10942 switch (uVector)
10943 {
10944 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
10945 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
10946 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
10947 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
10948 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
10949 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
10950
10951 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10952 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10953 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
10954 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10955 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
10956 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10957 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
10958 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10959 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
10960 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10961 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
10962 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10963 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
10964 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10965 default:
10966 {
10967 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
10968 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10969 {
10970 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
10971 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
10972 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10973
10974 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
10975 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10976 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10977 AssertRCReturn(rc, rc);
10978 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
10979 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
10980 0 /* GCPtrFaultAddress */);
10981 }
10982 else
10983 {
10984 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
10985 pVCpu->hm.s.u32HMError = uVector;
10986 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10987 }
10988 break;
10989 }
10990 }
10991 break;
10992 }
10993
10994 default:
10995 {
10996 pVCpu->hm.s.u32HMError = uExitIntInfo;
10997 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
10998 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
10999 break;
11000 }
11001 }
11002 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11003 return rc;
11004}
11005
11006
11007/**
11008 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11009 */
11010HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11011{
11012 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11013
11014 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11015 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11016
11017 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11018 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11019 return VINF_SUCCESS;
11020}
11021
11022
11023/**
11024 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11025 */
11026HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11027{
11028 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11029 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11030 {
11031 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11032 HMVMX_RETURN_UNEXPECTED_EXIT();
11033 }
11034
11035 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11036
11037 /*
11038 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11039 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11040 */
11041 uint32_t fIntrState = 0;
11042 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11043 AssertRCReturn(rc, rc);
11044
11045 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11046 if ( fBlockSti
11047 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11048 {
11049 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11050 }
11051
11052 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11053 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11054
11055 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11056 return VINF_SUCCESS;
11057}
11058
11059
11060/**
11061 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11062 */
11063HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11064{
11065 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11066 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11067}
11068
11069
11070/**
11071 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11072 */
11073HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11074{
11075 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11076 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11077}
11078
11079
11080/**
11081 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11082 */
11083HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11084{
11085 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11086 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
11087
11088 /*
11089 * Get the state we need and update the exit history entry.
11090 */
11091 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11092 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
11093 | CPUMCTX_EXTRN_CS);
11094 AssertRCReturn(rc, rc);
11095
11096 VBOXSTRICTRC rcStrict;
11097 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11098 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
11099 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11100 if (!pExitRec)
11101 {
11102 /*
11103 * Regular CPUID instruction execution.
11104 */
11105 PVM pVM = pVCpu->CTX_SUFF(pVM);
11106 rcStrict = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11107 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11108 {
11109 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11110 Assert(pVmxTransient->cbInstr == 2);
11111 }
11112 else
11113 {
11114 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11115 rcStrict = VERR_EM_INTERPRETER;
11116 }
11117 }
11118 else
11119 {
11120 /*
11121 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
11122 */
11123 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
11124 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11125 AssertRCReturn(rc2, rc2);
11126
11127 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11128 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11129
11130 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11131 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11132
11133 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11134 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11135 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11136 }
11137 return VBOXSTRICTRC_TODO(rcStrict);
11138}
11139
11140
11141/**
11142 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11143 */
11144HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11145{
11146 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11147 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4);
11148 AssertRCReturn(rc, rc);
11149
11150 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11151 return VINF_EM_RAW_EMULATE_INSTR;
11152
11153 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11154 HMVMX_RETURN_UNEXPECTED_EXIT();
11155}
11156
11157
11158/**
11159 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11160 */
11161HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11162{
11163 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11164 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11165 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11166 AssertRCReturn(rc, rc);
11167
11168 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11169 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11170 {
11171 /* If we get a spurious VM-exit when offsetting is enabled,
11172 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11173 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11174 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11175 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
11176 | HM_CHANGED_GUEST_RFLAGS);
11177 }
11178 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11179 {
11180 rcStrict = VINF_SUCCESS;
11181 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11182 }
11183 return rcStrict;
11184}
11185
11186
11187/**
11188 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11189 */
11190HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11191{
11192 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11193 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
11194 | CPUMCTX_EXTRN_TSC_AUX);
11195 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11196 AssertRCReturn(rc, rc);
11197
11198 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11199 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11200 {
11201 /* If we get a spurious VM-exit when offsetting is enabled,
11202 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11203 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11204 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11205 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
11206 | HM_CHANGED_GUEST_RFLAGS);
11207 }
11208 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11209 {
11210 rcStrict = VINF_SUCCESS;
11211 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11212 }
11213 return rcStrict;
11214}
11215
11216
11217/**
11218 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11219 */
11220HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11221{
11222 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11223 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4
11224 | CPUMCTX_EXTRN_CR0
11225 | CPUMCTX_EXTRN_RFLAGS
11226 | CPUMCTX_EXTRN_SS);
11227 AssertRCReturn(rc, rc);
11228
11229 PVM pVM = pVCpu->CTX_SUFF(pVM);
11230 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11231 if (RT_LIKELY(rc == VINF_SUCCESS))
11232 {
11233 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11234 Assert(pVmxTransient->cbInstr == 2);
11235 }
11236 else
11237 {
11238 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11239 rc = VERR_EM_INTERPRETER;
11240 }
11241 return rc;
11242}
11243
11244
11245/**
11246 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11247 */
11248HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11249{
11250 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11251
11252 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11253 if (EMAreHypercallInstructionsEnabled(pVCpu))
11254 {
11255 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
11256 | CPUMCTX_EXTRN_RFLAGS
11257 | CPUMCTX_EXTRN_CR0
11258 | CPUMCTX_EXTRN_SS
11259 | CPUMCTX_EXTRN_CS
11260 | CPUMCTX_EXTRN_EFER);
11261 AssertRCReturn(rc, rc);
11262
11263 /* Perform the hypercall. */
11264 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11265 if (rcStrict == VINF_SUCCESS)
11266 {
11267 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11268 AssertRCReturn(rc, rc);
11269 }
11270 else
11271 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11272 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11273 || RT_FAILURE(rcStrict));
11274
11275 /* If the hypercall changes anything other than guest's general-purpose registers,
11276 we would need to reload the guest changed bits here before VM-entry. */
11277 }
11278 else
11279 Log4Func(("Hypercalls not enabled\n"));
11280
11281 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11282 if (RT_FAILURE(rcStrict))
11283 {
11284 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11285 rcStrict = VINF_SUCCESS;
11286 }
11287
11288 return rcStrict;
11289}
11290
11291
11292/**
11293 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11294 */
11295HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11296{
11297 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11298 PVM pVM = pVCpu->CTX_SUFF(pVM);
11299 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11300
11301 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11302 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK);
11303 AssertRCReturn(rc, rc);
11304
11305 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11306 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11307 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11308 else
11309 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11310 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11311 return rcStrict;
11312}
11313
11314
11315/**
11316 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11317 */
11318HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11319{
11320 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11321 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11322 | CPUMCTX_EXTRN_RFLAGS
11323 | CPUMCTX_EXTRN_SS);
11324 AssertRCReturn(rc, rc);
11325
11326 PVM pVM = pVCpu->CTX_SUFF(pVM);
11327 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11328 if (RT_LIKELY(rc == VINF_SUCCESS))
11329 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11330 else
11331 {
11332 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11333 rc = VERR_EM_INTERPRETER;
11334 }
11335 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11336 return rc;
11337}
11338
11339
11340/**
11341 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11342 */
11343HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11344{
11345 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11346 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11347 | CPUMCTX_EXTRN_RFLAGS
11348 | CPUMCTX_EXTRN_SS);
11349 AssertRCReturn(rc, rc);
11350
11351 PVM pVM = pVCpu->CTX_SUFF(pVM);
11352 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11353 rc = VBOXSTRICTRC_VAL(rc2);
11354 if (RT_LIKELY( rc == VINF_SUCCESS
11355 || rc == VINF_EM_HALT))
11356 {
11357 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11358 AssertRCReturn(rc3, rc3);
11359
11360 if ( rc == VINF_EM_HALT
11361 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11362 rc = VINF_SUCCESS;
11363 }
11364 else
11365 {
11366 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11367 rc = VERR_EM_INTERPRETER;
11368 }
11369 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11370 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11371 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11372 return rc;
11373}
11374
11375
11376/**
11377 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11378 */
11379HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11380{
11381 /*
11382 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11383 * mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11384 * treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11385 * VMX root operation. If we get here, something funny is going on.
11386 *
11387 * See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11388 */
11389 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11390 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11391 HMVMX_RETURN_UNEXPECTED_EXIT();
11392}
11393
11394
11395/**
11396 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11397 */
11398HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11399{
11400 /*
11401 * This can only happen if we support dual-monitor treatment of SMI, which can be activated
11402 * by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11403 * this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11404 * an SMI. If we get here, something funny is going on.
11405 *
11406 * See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11407 * See Intel spec. 25.3 "Other Causes of VM-Exits"
11408 */
11409 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11410 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11411 HMVMX_RETURN_UNEXPECTED_EXIT();
11412}
11413
11414
11415/**
11416 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11417 */
11418HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11419{
11420 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11421 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11422 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11423 HMVMX_RETURN_UNEXPECTED_EXIT();
11424}
11425
11426
11427/**
11428 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11429 */
11430HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11431{
11432 /*
11433 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11434 * We don't make use of it as our guests don't have direct access to the host LAPIC.
11435 * See Intel spec. 25.3 "Other Causes of VM-exits".
11436 */
11437 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11438 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11439 HMVMX_RETURN_UNEXPECTED_EXIT();
11440}
11441
11442
11443/**
11444 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11445 * VM-exit.
11446 */
11447HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11448{
11449 /*
11450 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11451 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11452 *
11453 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11454 * See Intel spec. "23.8 Restrictions on VMX operation".
11455 */
11456 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11457 return VINF_SUCCESS;
11458}
11459
11460
11461/**
11462 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11463 * VM-exit.
11464 */
11465HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11466{
11467 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11468 return VINF_EM_RESET;
11469}
11470
11471
11472/**
11473 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11474 */
11475HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11476{
11477 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11478 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11479
11480 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11481 AssertRCReturn(rc, rc);
11482
11483 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11484 rc = VINF_SUCCESS;
11485 else
11486 rc = VINF_EM_HALT;
11487
11488 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11489 if (rc != VINF_SUCCESS)
11490 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11491 return rc;
11492}
11493
11494
11495/**
11496 * VM-exit handler for instructions that result in a \#UD exception delivered to
11497 * the guest.
11498 */
11499HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11500{
11501 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11502 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11503 return VINF_SUCCESS;
11504}
11505
11506
11507/**
11508 * VM-exit handler for expiry of the VMX preemption timer.
11509 */
11510HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11511{
11512 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11513
11514 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11515 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11516
11517 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11518 PVM pVM = pVCpu->CTX_SUFF(pVM);
11519 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11520 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11521 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11522}
11523
11524
11525/**
11526 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11527 */
11528HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11529{
11530 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11531
11532 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11533 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
11534 | CPUMCTX_EXTRN_CR4);
11535 AssertRCReturn(rc, rc);
11536
11537 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11538 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11539 : HM_CHANGED_XCPT_RAISED_MASK);
11540
11541 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11542
11543 return rcStrict;
11544}
11545
11546
11547/**
11548 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11549 */
11550HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11551{
11552 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11553 /** @todo Use VM-exit instruction information. */
11554 return VERR_EM_INTERPRETER;
11555}
11556
11557
11558/**
11559 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11560 * Error VM-exit.
11561 */
11562HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11563{
11564 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11565 rc |= hmR0VmxCheckVmcsCtls(pVCpu);
11566 AssertRCReturn(rc, rc);
11567
11568 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pMixedCtx);
11569 NOREF(uInvalidReason);
11570
11571#ifdef VBOX_STRICT
11572 uint32_t fIntrState;
11573 RTHCUINTREG uHCReg;
11574 uint64_t u64Val;
11575 uint32_t u32Val;
11576
11577 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11578 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11579 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11580 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11581 AssertRCReturn(rc, rc);
11582
11583 Log4(("uInvalidReason %u\n", uInvalidReason));
11584 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11585 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11586 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11587 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", fIntrState));
11588
11589 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11590 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11591 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11592 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11593 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11594 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11595 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11596 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11597 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11598 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11599 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11600 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11601
11602 hmR0DumpRegs(pVCpu, pMixedCtx);
11603#else
11604 NOREF(pVmxTransient);
11605#endif
11606
11607 return VERR_VMX_INVALID_GUEST_STATE;
11608}
11609
11610
11611/**
11612 * VM-exit handler for VM-entry failure due to an MSR-load
11613 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11614 */
11615HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11616{
11617 NOREF(pVmxTransient);
11618 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11619 HMVMX_RETURN_UNEXPECTED_EXIT();
11620}
11621
11622
11623/**
11624 * VM-exit handler for VM-entry failure due to a machine-check event
11625 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11626 */
11627HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11628{
11629 NOREF(pVmxTransient);
11630 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11631 HMVMX_RETURN_UNEXPECTED_EXIT();
11632}
11633
11634
11635/**
11636 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11637 * theory.
11638 */
11639HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11640{
11641 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
11642 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
11643 return VERR_VMX_UNDEFINED_EXIT_CODE;
11644}
11645
11646
11647/**
11648 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11649 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11650 * Conditional VM-exit.
11651 */
11652HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11653{
11654 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11655
11656 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11657 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11658 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11659 return VERR_EM_INTERPRETER;
11660 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11661 HMVMX_RETURN_UNEXPECTED_EXIT();
11662}
11663
11664
11665/**
11666 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11667 */
11668HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11669{
11670 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11671
11672 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11673 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11674 return VERR_EM_INTERPRETER;
11675 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11676 HMVMX_RETURN_UNEXPECTED_EXIT();
11677}
11678
11679
11680/**
11681 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11682 */
11683HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11684{
11685 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11686
11687 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
11688 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11689 | CPUMCTX_EXTRN_RFLAGS
11690 | CPUMCTX_EXTRN_SS);
11691 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11692 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
11693 AssertRCReturn(rc, rc);
11694 Log4Func(("ecx=%#RX32\n", pMixedCtx->ecx));
11695
11696#ifdef VBOX_STRICT
11697 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
11698 {
11699 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
11700 && pMixedCtx->ecx != MSR_K6_EFER)
11701 {
11702 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11703 pMixedCtx->ecx));
11704 HMVMX_RETURN_UNEXPECTED_EXIT();
11705 }
11706 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11707 {
11708 VMXMSREXITREAD enmRead;
11709 VMXMSREXITWRITE enmWrite;
11710 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
11711 AssertRCReturn(rc2, rc2);
11712 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11713 {
11714 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11715 HMVMX_RETURN_UNEXPECTED_EXIT();
11716 }
11717 }
11718 }
11719#endif
11720
11721 PVM pVM = pVCpu->CTX_SUFF(pVM);
11722 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11723 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
11724 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
11725 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11726 if (RT_SUCCESS(rc))
11727 {
11728 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11729 Assert(pVmxTransient->cbInstr == 2);
11730 }
11731 return rc;
11732}
11733
11734
11735/**
11736 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11737 */
11738HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11739{
11740 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11741 PVM pVM = pVCpu->CTX_SUFF(pVM);
11742 int rc = VINF_SUCCESS;
11743
11744 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
11745 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11746 | CPUMCTX_EXTRN_RFLAGS
11747 | CPUMCTX_EXTRN_SS);
11748 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11749 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
11750 AssertRCReturn(rc, rc);
11751 Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
11752
11753 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11754 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
11755 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
11756
11757 if (RT_SUCCESS(rc))
11758 {
11759 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11760
11761 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
11762 if ( pMixedCtx->ecx == MSR_IA32_APICBASE
11763 || ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
11764 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
11765 {
11766 /*
11767 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
11768 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
11769 * EMInterpretWrmsr() changes it.
11770 */
11771 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11772 }
11773 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
11774 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11775 else if (pMixedCtx->ecx == MSR_K6_EFER)
11776 {
11777 /*
11778 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
11779 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
11780 * the other bits as well, SCE and NXE. See @bugref{7368}.
11781 */
11782 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
11783 | HM_CHANGED_VMX_ENTRY_CTLS
11784 | HM_CHANGED_VMX_EXIT_CTLS);
11785 }
11786
11787 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
11788 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11789 {
11790 switch (pMixedCtx->ecx)
11791 {
11792 case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
11793 case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
11794 case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
11795 case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
11796 case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
11797 case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
11798 default:
11799 {
11800 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11801 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
11802 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11803 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
11804 break;
11805 }
11806 }
11807 }
11808#ifdef VBOX_STRICT
11809 else
11810 {
11811 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
11812 switch (pMixedCtx->ecx)
11813 {
11814 case MSR_IA32_SYSENTER_CS:
11815 case MSR_IA32_SYSENTER_EIP:
11816 case MSR_IA32_SYSENTER_ESP:
11817 case MSR_K8_FS_BASE:
11818 case MSR_K8_GS_BASE:
11819 {
11820 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
11821 HMVMX_RETURN_UNEXPECTED_EXIT();
11822 }
11823
11824 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
11825 default:
11826 {
11827 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11828 {
11829 /* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
11830 if (pMixedCtx->ecx != MSR_K6_EFER)
11831 {
11832 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11833 pMixedCtx->ecx));
11834 HMVMX_RETURN_UNEXPECTED_EXIT();
11835 }
11836 }
11837
11838 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11839 {
11840 VMXMSREXITREAD enmRead;
11841 VMXMSREXITWRITE enmWrite;
11842 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
11843 AssertRCReturn(rc2, rc2);
11844 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
11845 {
11846 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11847 HMVMX_RETURN_UNEXPECTED_EXIT();
11848 }
11849 }
11850 break;
11851 }
11852 }
11853 }
11854#endif /* VBOX_STRICT */
11855 }
11856 return rc;
11857}
11858
11859
11860/**
11861 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
11862 */
11863HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11864{
11865 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11866 /** @todo The guest has likely hit a contended spinlock. We might want to
11867 * poke a schedule different guest VCPU. */
11868 return VINF_EM_RAW_INTERRUPT;
11869}
11870
11871
11872/**
11873 * VM-exit handler for when the TPR value is lowered below the specified
11874 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
11875 */
11876HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11877{
11878 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11879 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
11880
11881 /*
11882 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
11883 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
11884 */
11885 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
11886 return VINF_SUCCESS;
11887}
11888
11889
11890/**
11891 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
11892 * VM-exit.
11893 *
11894 * @retval VINF_SUCCESS when guest execution can continue.
11895 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
11896 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
11897 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
11898 * interpreter.
11899 */
11900HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11901{
11902 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11903 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
11904
11905 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11906 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11907 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11908 AssertRCReturn(rc, rc);
11909
11910 VBOXSTRICTRC rcStrict;
11911 PVM pVM = pVCpu->CTX_SUFF(pVM);
11912 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
11913 uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQualification);
11914 switch (uAccessType)
11915 {
11916 case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
11917 {
11918 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
11919 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
11920 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification));
11921 AssertMsg( rcStrict == VINF_SUCCESS
11922 || rcStrict == VINF_IEM_RAISED_XCPT
11923 || rcStrict == VINF_PGM_CHANGE_MODE
11924 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11925
11926 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
11927 {
11928 case 0:
11929 {
11930 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11931 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
11932 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
11933 break;
11934 }
11935
11936 case 2:
11937 {
11938 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
11939 /* Nothing to do here, CR2 it's not part of the VMCS. */
11940 break;
11941 }
11942
11943 case 3:
11944 {
11945 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
11946 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
11947 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR3);
11948 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
11949 break;
11950 }
11951
11952 case 4:
11953 {
11954 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
11955 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11956 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
11957 pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
11958 break;
11959 }
11960
11961 case 8:
11962 {
11963 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
11964 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
11965 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11966 break;
11967 }
11968 default:
11969 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification)));
11970 break;
11971 }
11972 break;
11973 }
11974
11975 case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
11976 {
11977 Assert( !pVM->hm.s.fNestedPaging
11978 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
11979 || pVCpu->hm.s.fUsingDebugLoop
11980 || VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 3);
11981 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
11982 Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 8
11983 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
11984
11985 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
11986 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification),
11987 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification));
11988 AssertMsg( rcStrict == VINF_SUCCESS
11989 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11990#ifdef VBOX_WITH_STATISTICS
11991 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
11992 {
11993 case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
11994 case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
11995 case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
11996 case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
11997 case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
11998 }
11999#endif
12000 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
12001 VBOXSTRICTRC_VAL(rcStrict)));
12002 if (VMX_EXIT_QUAL_CRX_GENREG(uExitQualification) == X86_GREG_xSP)
12003 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RSP);
12004 break;
12005 }
12006
12007 case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12008 {
12009 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12010 AssertMsg( rcStrict == VINF_SUCCESS
12011 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12012
12013 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12014 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12015 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12016 break;
12017 }
12018
12019 case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12020 {
12021 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12022 VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQualification));
12023 AssertMsg( rcStrict == VINF_SUCCESS
12024 || rcStrict == VINF_IEM_RAISED_XCPT
12025 || rcStrict == VINF_PGM_CHANGE_MODE,
12026 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12027
12028 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12029 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12030 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12031 break;
12032 }
12033
12034 default:
12035 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12036 VERR_VMX_UNEXPECTED_EXCEPTION);
12037 }
12038
12039 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
12040 : HM_CHANGED_XCPT_RAISED_MASK);
12041 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12042 NOREF(pVM);
12043 return rcStrict;
12044}
12045
12046
12047/**
12048 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12049 * VM-exit.
12050 */
12051HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12052{
12053 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12054 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12055 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
12056
12057 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12058 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12059 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
12060 | CPUMCTX_EXTRN_SREG_MASK
12061 | CPUMCTX_EXTRN_EFER);
12062 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12063 AssertRCReturn(rc, rc);
12064
12065 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12066 uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQualification);
12067 uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQualification);
12068 bool fIOWrite = ( VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQualification)
12069 == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12070 bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQualification);
12071 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12072 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12073 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12074
12075 /*
12076 * Update exit history to see if this exit can be optimized.
12077 */
12078 VBOXSTRICTRC rcStrict;
12079 PCEMEXITREC pExitRec = NULL;
12080 if ( !fGstStepping
12081 && !fDbgStepping)
12082 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12083 !fIOString
12084 ? !fIOWrite
12085 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12086 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12087 : !fIOWrite
12088 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12089 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12090 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12091 if (!pExitRec)
12092 {
12093 /* I/O operation lookup arrays. */
12094 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12095 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12096 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12097 uint32_t const cbInstr = pVmxTransient->cbInstr;
12098 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12099 PVM pVM = pVCpu->CTX_SUFF(pVM);
12100 if (fIOString)
12101 {
12102 /*
12103 * INS/OUTS - I/O String instruction.
12104 *
12105 * Use instruction-information if available, otherwise fall back on
12106 * interpreting the instruction.
12107 */
12108 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12109 fIOWrite ? 'w' : 'r'));
12110 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12111 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12112 {
12113 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12114 AssertRCReturn(rc2, rc2);
12115 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12116 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12117 IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12118 bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification);
12119 if (fIOWrite)
12120 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12121 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12122 else
12123 {
12124 /*
12125 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12126 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12127 * See Intel Instruction spec. for "INS".
12128 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12129 */
12130 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12131 }
12132 }
12133 else
12134 rcStrict = IEMExecOne(pVCpu);
12135
12136 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12137 fUpdateRipAlready = true;
12138 }
12139 else
12140 {
12141 /*
12142 * IN/OUT - I/O instruction.
12143 */
12144 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12145 fIOWrite ? 'w' : 'r'));
12146 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12147 Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification));
12148 if (fIOWrite)
12149 {
12150 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12151 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12152 }
12153 else
12154 {
12155 uint32_t u32Result = 0;
12156 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12157 if (IOM_SUCCESS(rcStrict))
12158 {
12159 /* Save result of I/O IN instr. in AL/AX/EAX. */
12160 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12161 }
12162 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12163 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12164 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12165 }
12166 }
12167
12168 if (IOM_SUCCESS(rcStrict))
12169 {
12170 if (!fUpdateRipAlready)
12171 {
12172 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12173 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12174 }
12175
12176 /*
12177 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12178 * while booting Fedora 17 64-bit guest.
12179 *
12180 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12181 */
12182 if (fIOString)
12183 {
12184 /** @todo Single-step for INS/OUTS with REP prefix? */
12185 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12186 }
12187 else if ( !fDbgStepping
12188 && fGstStepping)
12189 {
12190 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
12191 AssertRCReturn(rc, rc);
12192 }
12193
12194 /*
12195 * If any I/O breakpoints are armed, we need to check if one triggered
12196 * and take appropriate action.
12197 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12198 */
12199 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12200 AssertRCReturn(rc, rc);
12201
12202 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12203 * execution engines about whether hyper BPs and such are pending. */
12204 uint32_t const uDr7 = pMixedCtx->dr[7];
12205 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12206 && X86_DR7_ANY_RW_IO(uDr7)
12207 && (pMixedCtx->cr4 & X86_CR4_DE))
12208 || DBGFBpIsHwIoArmed(pVM)))
12209 {
12210 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12211
12212 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12213 VMMRZCallRing3Disable(pVCpu);
12214 HM_DISABLE_PREEMPT();
12215
12216 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12217
12218 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12219 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12220 {
12221 /* Raise #DB. */
12222 if (fIsGuestDbgActive)
12223 ASMSetDR6(pMixedCtx->dr[6]);
12224 if (pMixedCtx->dr[7] != uDr7)
12225 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR7;
12226
12227 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12228 }
12229 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12230 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12231 else if ( rcStrict2 != VINF_SUCCESS
12232 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12233 rcStrict = rcStrict2;
12234 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12235
12236 HM_RESTORE_PREEMPT();
12237 VMMRZCallRing3Enable(pVCpu);
12238 }
12239 }
12240
12241#ifdef VBOX_STRICT
12242 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12243 Assert(!fIOWrite);
12244 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12245 Assert(fIOWrite);
12246 else
12247 {
12248# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12249 * statuses, that the VMM device and some others may return. See
12250 * IOM_SUCCESS() for guidance. */
12251 AssertMsg( RT_FAILURE(rcStrict)
12252 || rcStrict == VINF_SUCCESS
12253 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12254 || rcStrict == VINF_EM_DBG_BREAKPOINT
12255 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12256 || rcStrict == VINF_EM_RAW_TO_R3
12257 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12258# endif
12259 }
12260#endif
12261 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12262 }
12263 else
12264 {
12265 /*
12266 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12267 */
12268 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12269 AssertRCReturn(rc2, rc2);
12270 STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12271 : fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12272 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12273 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12274 VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification) ? "REP " : "",
12275 fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12276
12277 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12278 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12279
12280 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12281 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12282 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12283 }
12284 return rcStrict;
12285}
12286
12287
12288/**
12289 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12290 * VM-exit.
12291 */
12292HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12293{
12294 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12295
12296 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12297 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12298 AssertRCReturn(rc, rc);
12299 if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12300 {
12301 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12302 AssertRCReturn(rc, rc);
12303 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12304 {
12305 uint32_t uErrCode;
12306 RTGCUINTPTR GCPtrFaultAddress;
12307 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12308 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12309 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12310 if (fErrorCodeValid)
12311 {
12312 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12313 AssertRCReturn(rc, rc);
12314 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12315 }
12316 else
12317 uErrCode = 0;
12318
12319 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12320 && uVector == X86_XCPT_PF)
12321 GCPtrFaultAddress = pMixedCtx->cr2;
12322 else
12323 GCPtrFaultAddress = 0;
12324
12325 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12326 0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12327
12328 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12329 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12330 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12331 }
12332 }
12333
12334 /* Fall back to the interpreter to emulate the task-switch. */
12335 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12336 return VERR_EM_INTERPRETER;
12337}
12338
12339
12340/**
12341 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12342 */
12343HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12344{
12345 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12346 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12347 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12348 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12349 AssertRCReturn(rc, rc);
12350 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12351 return VINF_EM_DBG_STEPPED;
12352}
12353
12354
12355/**
12356 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12357 */
12358HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12359{
12360 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12361
12362 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12363
12364 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12365 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12366 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12367 {
12368 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12369 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12370 {
12371 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12372 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12373 }
12374 }
12375 else
12376 {
12377 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12378 rcStrict1 = VINF_SUCCESS;
12379 return rcStrict1;
12380 }
12381
12382 /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12383 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12384 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12385 AssertRCReturn(rc, rc);
12386
12387 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12388 uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12389 VBOXSTRICTRC rcStrict2;
12390 switch (uAccessType)
12391 {
12392 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12393 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12394 {
12395 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12396 || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
12397 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12398
12399 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12400 GCPhys &= PAGE_BASE_GC_MASK;
12401 GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12402 PVM pVM = pVCpu->CTX_SUFF(pVM);
12403 Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12404 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12405
12406 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12407 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12408 CPUMCTX2CORE(pMixedCtx), GCPhys);
12409 Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12410 if ( rcStrict2 == VINF_SUCCESS
12411 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12412 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12413 {
12414 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12415 | HM_CHANGED_GUEST_RSP
12416 | HM_CHANGED_GUEST_RFLAGS
12417 | HM_CHANGED_GUEST_APIC_TPR);
12418 rcStrict2 = VINF_SUCCESS;
12419 }
12420 break;
12421 }
12422
12423 default:
12424 Log4Func(("uAccessType=%#x\n", uAccessType));
12425 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12426 break;
12427 }
12428
12429 if (rcStrict2 != VINF_SUCCESS)
12430 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12431 return rcStrict2;
12432}
12433
12434
12435/**
12436 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12437 * VM-exit.
12438 */
12439HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12440{
12441 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12442
12443 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12444 if (pVmxTransient->fWasGuestDebugStateActive)
12445 {
12446 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12447 HMVMX_RETURN_UNEXPECTED_EXIT();
12448 }
12449
12450 if ( !pVCpu->hm.s.fSingleInstruction
12451 && !pVmxTransient->fWasHyperDebugStateActive)
12452 {
12453 Assert(!DBGFIsStepping(pVCpu));
12454 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12455
12456 /* Don't intercept MOV DRx any more. */
12457 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12458 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12459 AssertRCReturn(rc, rc);
12460
12461 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12462 VMMRZCallRing3Disable(pVCpu);
12463 HM_DISABLE_PREEMPT();
12464
12465 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12466 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12467 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12468
12469 HM_RESTORE_PREEMPT();
12470 VMMRZCallRing3Enable(pVCpu);
12471
12472#ifdef VBOX_WITH_STATISTICS
12473 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12474 AssertRCReturn(rc, rc);
12475 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12476 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12477 else
12478 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12479#endif
12480 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12481 return VINF_SUCCESS;
12482 }
12483
12484 /*
12485 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12486 * Update the segment registers and DR7 from the CPU.
12487 */
12488 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12489 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK
12490 | CPUMCTX_EXTRN_DR7);
12491 AssertRCReturn(rc, rc);
12492 Log4Func(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12493
12494 PVM pVM = pVCpu->CTX_SUFF(pVM);
12495 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12496 {
12497 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12498 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification),
12499 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification));
12500 if (RT_SUCCESS(rc))
12501 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12502 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12503 }
12504 else
12505 {
12506 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12507 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification),
12508 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification));
12509 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12510 }
12511
12512 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12513 if (RT_SUCCESS(rc))
12514 {
12515 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12516 AssertRCReturn(rc2, rc2);
12517 return VINF_SUCCESS;
12518 }
12519 return rc;
12520}
12521
12522
12523/**
12524 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12525 * Conditional VM-exit.
12526 */
12527HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12528{
12529 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12530 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12531
12532 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12533 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12534 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12535 {
12536 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12537 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12538 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12539 {
12540 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12541 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12542 }
12543 }
12544 else
12545 {
12546 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12547 rcStrict1 = VINF_SUCCESS;
12548 return rcStrict1;
12549 }
12550
12551 /*
12552 * Get sufficent state and update the exit history entry.
12553 */
12554 RTGCPHYS GCPhys;
12555 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12556 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12557 AssertRCReturn(rc, rc);
12558
12559 VBOXSTRICTRC rcStrict;
12560 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12561 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
12562 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12563 if (!pExitRec)
12564 {
12565 /*
12566 * If we succeed, resume guest execution.
12567 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12568 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12569 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12570 * weird case. See @bugref{6043}.
12571 */
12572 PVM pVM = pVCpu->CTX_SUFF(pVM);
12573 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12574 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12575 if ( rcStrict == VINF_SUCCESS
12576 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12577 || rcStrict == VERR_PAGE_NOT_PRESENT)
12578 {
12579 /* Successfully handled MMIO operation. */
12580 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12581 | HM_CHANGED_GUEST_RSP
12582 | HM_CHANGED_GUEST_RFLAGS
12583 | HM_CHANGED_GUEST_APIC_TPR);
12584 rcStrict = VINF_SUCCESS;
12585 }
12586 }
12587 else
12588 {
12589 /*
12590 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12591 */
12592 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
12593 int rc2 = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12594 AssertRCReturn(rc2, rc2);
12595
12596 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12597 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12598
12599 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12600 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12601
12602 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12603 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12604 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12605 }
12606 return VBOXSTRICTRC_TODO(rcStrict);
12607}
12608
12609
12610/**
12611 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12612 * VM-exit.
12613 */
12614HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12615{
12616 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12617 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12618
12619 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12620 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12621 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12622 {
12623 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12624 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12625 Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12626 }
12627 else
12628 {
12629 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12630 rcStrict1 = VINF_SUCCESS;
12631 return rcStrict1;
12632 }
12633
12634 RTGCPHYS GCPhys;
12635 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12636 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12637 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12638 AssertRCReturn(rc, rc);
12639
12640 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12641 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12642
12643 RTGCUINT uErrorCode = 0;
12644 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12645 uErrorCode |= X86_TRAP_PF_ID;
12646 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12647 uErrorCode |= X86_TRAP_PF_RW;
12648 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12649 uErrorCode |= X86_TRAP_PF_P;
12650
12651 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12652
12653 Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12654 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12655
12656 /* Handle the pagefault trap for the nested shadow table. */
12657 PVM pVM = pVCpu->CTX_SUFF(pVM);
12658 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12659 TRPMResetTrap(pVCpu);
12660
12661 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12662 if ( rcStrict2 == VINF_SUCCESS
12663 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12664 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12665 {
12666 /* Successfully synced our nested page tables. */
12667 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12668 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12669 | HM_CHANGED_GUEST_RSP
12670 | HM_CHANGED_GUEST_RFLAGS);
12671 return VINF_SUCCESS;
12672 }
12673
12674 Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12675 return rcStrict2;
12676}
12677
12678/** @} */
12679
12680/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12681/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
12682/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12683
12684/** @name VM-exit exception handlers.
12685 * @{
12686 */
12687
12688/**
12689 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12690 */
12691static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12692{
12693 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12694 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12695
12696 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
12697 AssertRCReturn(rc, rc);
12698
12699 if (!(pMixedCtx->cr0 & X86_CR0_NE))
12700 {
12701 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12702 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12703
12704 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12705 * provides VM-exit instruction length. If this causes problem later,
12706 * disassemble the instruction like it's done on AMD-V. */
12707 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12708 AssertRCReturn(rc2, rc2);
12709 return rc;
12710 }
12711
12712 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12713 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12714 return rc;
12715}
12716
12717
12718/**
12719 * VM-exit exception handler for \#BP (Breakpoint exception).
12720 */
12721static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12722{
12723 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12724 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12725
12726 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12727 AssertRCReturn(rc, rc);
12728
12729 rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx));
12730 if (rc == VINF_EM_RAW_GUEST_TRAP)
12731 {
12732 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12733 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12734 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12735 AssertRCReturn(rc, rc);
12736
12737 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12738 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12739 }
12740
12741 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
12742 return rc;
12743}
12744
12745
12746/**
12747 * VM-exit exception handler for \#AC (alignment check exception).
12748 */
12749static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12750{
12751 RT_NOREF_PV(pMixedCtx);
12752 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12753
12754 /*
12755 * Re-inject it. We'll detect any nesting before getting here.
12756 */
12757 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12758 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12759 AssertRCReturn(rc, rc);
12760 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
12761
12762 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12763 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12764 return VINF_SUCCESS;
12765}
12766
12767
12768/**
12769 * VM-exit exception handler for \#DB (Debug exception).
12770 */
12771static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12772{
12773 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12774 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
12775
12776 /*
12777 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
12778 * for processing.
12779 */
12780 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12781
12782 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
12783 uint64_t uDR6 = X86_DR6_INIT_VAL;
12784 uDR6 |= ( pVmxTransient->uExitQualification
12785 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
12786
12787 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
12788 Log6Func(("rc=%Rrc\n", rc));
12789 if (rc == VINF_EM_RAW_GUEST_TRAP)
12790 {
12791 /*
12792 * The exception was for the guest. Update DR6, DR7.GD and
12793 * IA32_DEBUGCTL.LBR before forwarding it.
12794 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
12795 */
12796 VMMRZCallRing3Disable(pVCpu);
12797 HM_DISABLE_PREEMPT();
12798
12799 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
12800 pMixedCtx->dr[6] |= uDR6;
12801 if (CPUMIsGuestDebugStateActive(pVCpu))
12802 ASMSetDR6(pMixedCtx->dr[6]);
12803
12804 HM_RESTORE_PREEMPT();
12805 VMMRZCallRing3Enable(pVCpu);
12806
12807 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12808 AssertRCReturn(rc, rc);
12809
12810 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
12811 pMixedCtx->dr[7] &= ~X86_DR7_GD;
12812
12813 /* Paranoia. */
12814 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
12815 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
12816
12817 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
12818 AssertRCReturn(rc, rc);
12819
12820 /*
12821 * Raise #DB in the guest.
12822 *
12823 * It is important to reflect exactly what the VM-exit gave us (preserving the
12824 * interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
12825 * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
12826 * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
12827 *
12828 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
12829 * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
12830 */
12831 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12832 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12833 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12834 AssertRCReturn(rc, rc);
12835 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12836 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12837 return VINF_SUCCESS;
12838 }
12839
12840 /*
12841 * Not a guest trap, must be a hypervisor related debug event then.
12842 * Update DR6 in case someone is interested in it.
12843 */
12844 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
12845 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
12846 CPUMSetHyperDR6(pVCpu, uDR6);
12847
12848 return rc;
12849}
12850
12851/**
12852 * VM-exit exception handler for \#GP (General-protection exception).
12853 *
12854 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
12855 */
12856static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12857{
12858 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12859 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
12860
12861 int rc;
12862 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
12863 { /* likely */ }
12864 else
12865 {
12866#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12867 Assert(pVCpu->hm.s.fUsingDebugLoop);
12868#endif
12869 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
12870 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12871 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12872 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12873 rc |= hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12874 AssertRCReturn(rc, rc);
12875 Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
12876 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
12877 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12878 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12879 return rc;
12880 }
12881
12882 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
12883 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
12884
12885 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
12886 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12887 AssertRCReturn(rc, rc);
12888
12889 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12890 uint32_t cbOp = 0;
12891 PVM pVM = pVCpu->CTX_SUFF(pVM);
12892 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12893 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
12894 if (RT_SUCCESS(rc))
12895 {
12896 rc = VINF_SUCCESS;
12897 Assert(cbOp == pDis->cbInstr);
12898 Log4Func(("Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12899 switch (pDis->pCurInstr->uOpcode)
12900 {
12901 case OP_CLI:
12902 {
12903 pMixedCtx->eflags.Bits.u1IF = 0;
12904 pMixedCtx->eflags.Bits.u1RF = 0;
12905 pMixedCtx->rip += pDis->cbInstr;
12906 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12907 if ( !fDbgStepping
12908 && pMixedCtx->eflags.Bits.u1TF)
12909 {
12910 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
12911 AssertRCReturn(rc, rc);
12912 }
12913 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
12914 break;
12915 }
12916
12917 case OP_STI:
12918 {
12919 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
12920 pMixedCtx->eflags.Bits.u1IF = 1;
12921 pMixedCtx->eflags.Bits.u1RF = 0;
12922 pMixedCtx->rip += pDis->cbInstr;
12923 if (!fOldIF)
12924 {
12925 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
12926 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
12927 }
12928 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12929 if ( !fDbgStepping
12930 && pMixedCtx->eflags.Bits.u1TF)
12931 {
12932 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
12933 AssertRCReturn(rc, rc);
12934 }
12935 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
12936 break;
12937 }
12938
12939 case OP_HLT:
12940 {
12941 rc = VINF_EM_HALT;
12942 pMixedCtx->rip += pDis->cbInstr;
12943 pMixedCtx->eflags.Bits.u1RF = 0;
12944 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12945 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
12946 break;
12947 }
12948
12949 case OP_POPF:
12950 {
12951 Log4Func(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12952 uint32_t cbParm;
12953 uint32_t uMask;
12954 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12955 if (pDis->fPrefix & DISPREFIX_OPSIZE)
12956 {
12957 cbParm = 4;
12958 uMask = 0xffffffff;
12959 }
12960 else
12961 {
12962 cbParm = 2;
12963 uMask = 0xffff;
12964 }
12965
12966 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
12967 RTGCPTR GCPtrStack = 0;
12968 X86EFLAGS Eflags;
12969 Eflags.u32 = 0;
12970 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
12971 &GCPtrStack);
12972 if (RT_SUCCESS(rc))
12973 {
12974 Assert(sizeof(Eflags.u32) >= cbParm);
12975 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
12976 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
12977 }
12978 if (RT_FAILURE(rc))
12979 {
12980 rc = VERR_EM_INTERPRETER;
12981 break;
12982 }
12983 Log4Func(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
12984 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
12985 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
12986 pMixedCtx->esp += cbParm;
12987 pMixedCtx->esp &= uMask;
12988 pMixedCtx->rip += pDis->cbInstr;
12989 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12990 | HM_CHANGED_GUEST_RSP
12991 | HM_CHANGED_GUEST_RFLAGS);
12992 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
12993 POPF restores EFLAGS.TF. */
12994 if ( !fDbgStepping
12995 && fGstStepping)
12996 {
12997 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
12998 AssertRCReturn(rc, rc);
12999 }
13000 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13001 break;
13002 }
13003
13004 case OP_PUSHF:
13005 {
13006 uint32_t cbParm;
13007 uint32_t uMask;
13008 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13009 {
13010 cbParm = 4;
13011 uMask = 0xffffffff;
13012 }
13013 else
13014 {
13015 cbParm = 2;
13016 uMask = 0xffff;
13017 }
13018
13019 /* Get the stack pointer & push the contents of eflags onto the stack. */
13020 RTGCPTR GCPtrStack = 0;
13021 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13022 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13023 if (RT_FAILURE(rc))
13024 {
13025 rc = VERR_EM_INTERPRETER;
13026 break;
13027 }
13028 X86EFLAGS Eflags = pMixedCtx->eflags;
13029 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13030 Eflags.Bits.u1RF = 0;
13031 Eflags.Bits.u1VM = 0;
13032
13033 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13034 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13035 {
13036 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13037 rc = VERR_EM_INTERPRETER;
13038 break;
13039 }
13040 Log4Func(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13041 pMixedCtx->esp -= cbParm;
13042 pMixedCtx->esp &= uMask;
13043 pMixedCtx->rip += pDis->cbInstr;
13044 pMixedCtx->eflags.Bits.u1RF = 0;
13045 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13046 | HM_CHANGED_GUEST_RSP
13047 | HM_CHANGED_GUEST_RFLAGS);
13048 if ( !fDbgStepping
13049 && pMixedCtx->eflags.Bits.u1TF)
13050 {
13051 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13052 AssertRCReturn(rc, rc);
13053 }
13054 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13055 break;
13056 }
13057
13058 case OP_IRET:
13059 {
13060 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13061 * instruction reference. */
13062 RTGCPTR GCPtrStack = 0;
13063 uint32_t uMask = 0xffff;
13064 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13065 uint16_t aIretFrame[3];
13066 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13067 {
13068 rc = VERR_EM_INTERPRETER;
13069 break;
13070 }
13071 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13072 &GCPtrStack);
13073 if (RT_SUCCESS(rc))
13074 {
13075 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13076 PGMACCESSORIGIN_HM));
13077 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13078 }
13079 if (RT_FAILURE(rc))
13080 {
13081 rc = VERR_EM_INTERPRETER;
13082 break;
13083 }
13084 pMixedCtx->eip = 0;
13085 pMixedCtx->ip = aIretFrame[0];
13086 pMixedCtx->cs.Sel = aIretFrame[1];
13087 pMixedCtx->cs.ValidSel = aIretFrame[1];
13088 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13089 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13090 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13091 pMixedCtx->sp += sizeof(aIretFrame);
13092 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13093 | HM_CHANGED_GUEST_CS
13094 | HM_CHANGED_GUEST_RSP
13095 | HM_CHANGED_GUEST_RFLAGS);
13096 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13097 if ( !fDbgStepping
13098 && fGstStepping)
13099 {
13100 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13101 AssertRCReturn(rc, rc);
13102 }
13103 Log4Func(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13104 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13105 break;
13106 }
13107
13108 case OP_INT:
13109 {
13110 uint16_t uVector = pDis->Param1.uValue & 0xff;
13111 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13112 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13113 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13114 break;
13115 }
13116
13117 case OP_INTO:
13118 {
13119 if (pMixedCtx->eflags.Bits.u1OF)
13120 {
13121 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13122 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13123 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13124 }
13125 else
13126 {
13127 pMixedCtx->eflags.Bits.u1RF = 0;
13128 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
13129 }
13130 break;
13131 }
13132
13133 default:
13134 {
13135 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13136 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13137 EMCODETYPE_SUPERVISOR);
13138 rc = VBOXSTRICTRC_VAL(rc2);
13139 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13140 /** @todo We have to set pending-debug exceptions here when the guest is
13141 * single-stepping depending on the instruction that was interpreted. */
13142 Log4Func(("#GP rc=%Rrc\n", rc));
13143 break;
13144 }
13145 }
13146 }
13147 else
13148 rc = VERR_EM_INTERPRETER;
13149
13150 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13151 ("#GP Unexpected rc=%Rrc\n", rc));
13152 return rc;
13153}
13154
13155
13156/**
13157 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13158 * the exception reported in the VMX transient structure back into the VM.
13159 *
13160 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13161 * up-to-date.
13162 */
13163static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13164{
13165 RT_NOREF_PV(pMixedCtx);
13166 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13167#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13168 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13169 ("uVector=%#x u32XcptBitmap=%#X32\n",
13170 VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13171#endif
13172
13173 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13174 hmR0VmxCheckExitDueToEventDelivery(). */
13175 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13176 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13177 AssertRCReturn(rc, rc);
13178 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13179
13180#ifdef DEBUG_ramshankar
13181 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS
13182 | CPUMCTX_EXTRN_RIP);
13183 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13184 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13185#endif
13186
13187 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13188 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13189 return VINF_SUCCESS;
13190}
13191
13192
13193/**
13194 * VM-exit exception handler for \#PF (Page-fault exception).
13195 */
13196static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13197{
13198 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13199 PVM pVM = pVCpu->CTX_SUFF(pVM);
13200 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13201 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13202 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13203 AssertRCReturn(rc, rc);
13204
13205 if (!pVM->hm.s.fNestedPaging)
13206 { /* likely */ }
13207 else
13208 {
13209#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13210 Assert(pVCpu->hm.s.fUsingDebugLoop);
13211#endif
13212 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13213 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13214 {
13215 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13216 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13217 }
13218 else
13219 {
13220 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13221 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13222 Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13223 }
13224 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13225 return rc;
13226 }
13227
13228 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13229 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13230 if (pVmxTransient->fVectoringPF)
13231 {
13232 Assert(pVCpu->hm.s.Event.fPending);
13233 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13234 }
13235
13236 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13237 AssertRCReturn(rc, rc);
13238
13239 Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13240 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13241
13242 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13243 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13244 (RTGCPTR)pVmxTransient->uExitQualification);
13245
13246 Log4Func(("#PF: rc=%Rrc\n", rc));
13247 if (rc == VINF_SUCCESS)
13248 {
13249 /*
13250 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13251 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13252 */
13253 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13254 TRPMResetTrap(pVCpu);
13255 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13256 return rc;
13257 }
13258
13259 if (rc == VINF_EM_RAW_GUEST_TRAP)
13260 {
13261 if (!pVmxTransient->fVectoringDoublePF)
13262 {
13263 /* It's a guest page fault and needs to be reflected to the guest. */
13264 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13265 TRPMResetTrap(pVCpu);
13266 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13267 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13268 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13269 }
13270 else
13271 {
13272 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13273 TRPMResetTrap(pVCpu);
13274 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13275 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13276 Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13277 }
13278
13279 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13280 return VINF_SUCCESS;
13281 }
13282
13283 TRPMResetTrap(pVCpu);
13284 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13285 return rc;
13286}
13287
13288/** @} */
13289
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette