VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 74545

Last change on this file since 74545 was 74545, checked in by vboxsync, 6 years ago

VMM/HMVMXR0: Nested VMX: bugref:9180 Move hmR0VmxGetMsrPermission to the all-context code, will be used from IEM for
upcoming changes.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 573.9 KB
Line 
1/* $Id: HMVMXR0.cpp 74545 2018-10-01 10:08:21Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/x86.h>
25#include <iprt/asm-amd64-x86.h>
26#include <iprt/thread.h>
27
28#include <VBox/vmm/pdmapi.h>
29#include <VBox/vmm/dbgf.h>
30#include <VBox/vmm/iem.h>
31#include <VBox/vmm/iom.h>
32#include <VBox/vmm/selm.h>
33#include <VBox/vmm/tm.h>
34#include <VBox/vmm/em.h>
35#include <VBox/vmm/gim.h>
36#include <VBox/vmm/apic.h>
37#ifdef VBOX_WITH_REM
38# include <VBox/vmm/rem.h>
39#endif
40#include "HMInternal.h"
41#include <VBox/vmm/vm.h>
42#include "HMVMXR0.h"
43#include "dtrace/VBoxVMM.h"
44
45#ifdef DEBUG_ramshankar
46# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
47# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
49# define HMVMX_ALWAYS_CHECK_GUEST_STATE
50# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
51# define HMVMX_ALWAYS_TRAP_PF
52# define HMVMX_ALWAYS_FLUSH_TLB
53# define HMVMX_ALWAYS_SWAP_EFER
54#endif
55
56
57/*********************************************************************************************************************************
58* Defined Constants And Macros *
59*********************************************************************************************************************************/
60/** Use the function table. */
61#define HMVMX_USE_FUNCTION_TABLE
62
63/** Determine which tagged-TLB flush handler to use. */
64#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
65#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
66#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
67#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
68
69/** @name HMVMX_READ_XXX
70 * Flags to skip redundant reads of some common VMCS fields that are not part of
71 * the guest-CPU or VCPU state but are needed while handling VM-exits.
72 */
73#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
74#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
75#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
76#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
77#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
78#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
79#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
80/** @} */
81
82/**
83 * States of the VMCS.
84 *
85 * This does not reflect all possible VMCS states but currently only those
86 * needed for maintaining the VMCS consistently even when thread-context hooks
87 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
88 */
89#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
90#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
91#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
92
93/**
94 * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
95 * guest using hardware-assisted VMX.
96 *
97 * This excludes state like GPRs (other than RSP) which are always are
98 * swapped and restored across the world-switch and also registers like EFER,
99 * MSR which cannot be modified by the guest without causing a VM-exit.
100 */
101#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
102 | CPUMCTX_EXTRN_RFLAGS \
103 | CPUMCTX_EXTRN_RSP \
104 | CPUMCTX_EXTRN_SREG_MASK \
105 | CPUMCTX_EXTRN_TABLE_MASK \
106 | CPUMCTX_EXTRN_KERNEL_GS_BASE \
107 | CPUMCTX_EXTRN_SYSCALL_MSRS \
108 | CPUMCTX_EXTRN_SYSENTER_MSRS \
109 | CPUMCTX_EXTRN_TSC_AUX \
110 | CPUMCTX_EXTRN_OTHER_MSRS \
111 | CPUMCTX_EXTRN_CR0 \
112 | CPUMCTX_EXTRN_CR3 \
113 | CPUMCTX_EXTRN_CR4 \
114 | CPUMCTX_EXTRN_DR7 \
115 | CPUMCTX_EXTRN_HM_VMX_MASK)
116
117/**
118 * Exception bitmap mask for real-mode guests (real-on-v86).
119 *
120 * We need to intercept all exceptions manually except:
121 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
122 * due to bugs in Intel CPUs.
123 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
124 * support.
125 */
126#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
127 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
128 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
129 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
130 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
131 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
132 | RT_BIT(X86_XCPT_XF))
133
134/** Maximum VM-instruction error number. */
135#define HMVMX_INSTR_ERROR_MAX 28
136
137/** Profiling macro. */
138#ifdef HM_PROFILE_EXIT_DISPATCH
139# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
140# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
141#else
142# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
143# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
144#endif
145
146/** Assert that preemption is disabled or covered by thread-context hooks. */
147#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
148 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
149
150/** Assert that we haven't migrated CPUs when thread-context hooks are not
151 * used. */
152#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
153 || (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
154 ("Illegal migration! Entered on CPU %u Current %u\n", \
155 (a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
156
157/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
158 * context. */
159#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
160 ("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
161 (a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
162
163/** Macro for importing guest state from the VMCS back into CPUMCTX (intended to be
164 * used only from VM-exit handlers). */
165#define HMVMX_CPUMCTX_IMPORT_STATE(a_pVCpu, a_fWhat) (hmR0VmxImportGuestState((a_pVCpu), (a_fWhat)))
166
167/** Helper macro for VM-exit handlers called unexpectedly. */
168#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
169 do { \
170 (a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
171 return VERR_VMX_UNEXPECTED_EXIT; \
172 } while (0)
173
174/** Macro for importing segment registers to the VMCS from the guest-CPU context. */
175#ifdef VMX_USE_CACHED_VMCS_ACCESSES
176# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
177 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
178 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
179#else
180# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
181 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
182 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
183#endif
184
185/** Macro for exporting segment registers to the VMCS from the guest-CPU context. */
186#define HMVMX_EXPORT_SREG(Sel, a_pCtxSelReg) \
187 hmR0VmxExportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
188 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
189
190#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
191/** Macro that does the necessary privilege checks and intercepted VM-exits for
192 * guests that attempted to execute a VMX instruction. */
193# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
194 do \
195 { \
196 VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
197 if (rcStrictTmp == VINF_SUCCESS) \
198 { /* likely */ } \
199 else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
200 { \
201 Assert((a_pVCpu)->hm.s.Event.fPending); \
202 Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
203 return VINF_SUCCESS; \
204 } \
205 else \
206 { \
207 int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
208 AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
209 } \
210 } while (0)
211
212/** Macro that decodes a memory operand for an instruction VM-exit. */
213# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
214 do \
215 { \
216 VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
217 (a_pGCPtrEffAddr)); \
218 if (rcStrictTmp == VINF_SUCCESS) \
219 { /* likely */ } \
220 else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
221 { \
222 uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
223 Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
224 return VINF_SUCCESS; \
225 } \
226 else \
227 { \
228 Log4Func(("hmR0VmxCheckExitDueToVmxInstr failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
229 return rcStrictTmp; \
230 } \
231 } while (0)
232
233#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
234
235
236/*********************************************************************************************************************************
237* Structures and Typedefs *
238*********************************************************************************************************************************/
239/**
240 * VMX transient state.
241 *
242 * A state structure for holding miscellaneous information across
243 * VMX non-root operation and restored after the transition.
244 */
245typedef struct VMXTRANSIENT
246{
247 /** The host's rflags/eflags. */
248 RTCCUINTREG fEFlags;
249#if HC_ARCH_BITS == 32
250 uint32_t u32Alignment0;
251#endif
252 /** The guest's TPR value used for TPR shadowing. */
253 uint8_t u8GuestTpr;
254 /** Alignment. */
255 uint8_t abAlignment0[7];
256
257 /** The basic VM-exit reason. */
258 uint16_t uExitReason;
259 /** Alignment. */
260 uint16_t u16Alignment0;
261 /** The VM-exit interruption error code. */
262 uint32_t uExitIntErrorCode;
263 /** The VM-exit exit code qualification. */
264 uint64_t uExitQual;
265
266 /** The VM-exit interruption-information field. */
267 uint32_t uExitIntInfo;
268 /** The VM-exit instruction-length field. */
269 uint32_t cbInstr;
270 /** The VM-exit instruction-information field. */
271 VMXEXITINSTRINFO ExitInstrInfo;
272 /** Whether the VM-entry failed or not. */
273 bool fVMEntryFailed;
274 /** Alignment. */
275 uint8_t abAlignment1[3];
276
277 /** The VM-entry interruption-information field. */
278 uint32_t uEntryIntInfo;
279 /** The VM-entry exception error code field. */
280 uint32_t uEntryXcptErrorCode;
281 /** The VM-entry instruction length field. */
282 uint32_t cbEntryInstr;
283
284 /** IDT-vectoring information field. */
285 uint32_t uIdtVectoringInfo;
286 /** IDT-vectoring error code. */
287 uint32_t uIdtVectoringErrorCode;
288
289 /** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
290 uint32_t fVmcsFieldsRead;
291
292 /** Whether the guest debug state was active at the time of VM-exit. */
293 bool fWasGuestDebugStateActive;
294 /** Whether the hyper debug state was active at the time of VM-exit. */
295 bool fWasHyperDebugStateActive;
296 /** Whether TSC-offsetting should be setup before VM-entry. */
297 bool fUpdateTscOffsettingAndPreemptTimer;
298 /** Whether the VM-exit was caused by a page-fault during delivery of a
299 * contributory exception or a page-fault. */
300 bool fVectoringDoublePF;
301 /** Whether the VM-exit was caused by a page-fault during delivery of an
302 * external interrupt or NMI. */
303 bool fVectoringPF;
304} VMXTRANSIENT;
305AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
306AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
307AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
308AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
309AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
310/** Pointer to VMX transient state. */
311typedef VMXTRANSIENT *PVMXTRANSIENT;
312
313/**
314 * Memory operand read or write access.
315 */
316typedef enum VMXMEMACCESS
317{
318 VMXMEMACCESS_READ = 0,
319 VMXMEMACCESS_WRITE = 1
320} VMXMEMACCESS;
321
322/**
323 * VMX VM-exit handler.
324 *
325 * @returns Strict VBox status code (i.e. informational status codes too).
326 * @param pVCpu The cross context virtual CPU structure.
327 * @param pVmxTransient Pointer to the VMX-transient structure.
328 */
329#ifndef HMVMX_USE_FUNCTION_TABLE
330typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
331#else
332typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
333/** Pointer to VM-exit handler. */
334typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
335#endif
336
337/**
338 * VMX VM-exit handler, non-strict status code.
339 *
340 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
341 *
342 * @returns VBox status code, no informational status code returned.
343 * @param pVCpu The cross context virtual CPU structure.
344 * @param pVmxTransient Pointer to the VMX-transient structure.
345 *
346 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
347 * use of that status code will be replaced with VINF_EM_SOMETHING
348 * later when switching over to IEM.
349 */
350#ifndef HMVMX_USE_FUNCTION_TABLE
351typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
352#else
353typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
354#endif
355
356
357/*********************************************************************************************************************************
358* Internal Functions *
359*********************************************************************************************************************************/
360static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush);
361static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr);
362static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
363static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
364static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
365 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
366#if HC_ARCH_BITS == 32
367static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
368#endif
369#ifndef HMVMX_USE_FUNCTION_TABLE
370DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
371# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
372# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
373#else
374# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
375# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
376#endif
377
378/** @name VM-exit handlers.
379 * @{
380 */
381static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
382static FNVMXEXITHANDLER hmR0VmxExitExtInt;
383static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
384static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
385static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
386static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
387static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
388static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
389static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
390static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
391static FNVMXEXITHANDLER hmR0VmxExitCpuid;
392static FNVMXEXITHANDLER hmR0VmxExitGetsec;
393static FNVMXEXITHANDLER hmR0VmxExitHlt;
394static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
395static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
396static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
397static FNVMXEXITHANDLER hmR0VmxExitVmcall;
398#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
399static FNVMXEXITHANDLER hmR0VmxExitVmclear;
400static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
401static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
402static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
403static FNVMXEXITHANDLER hmR0VmxExitVmread;
404static FNVMXEXITHANDLER hmR0VmxExitVmresume;
405static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
406static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
407static FNVMXEXITHANDLER hmR0VmxExitVmxon;
408#endif
409static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
410static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
411static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
412static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
413static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
414static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
415static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
416static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
417static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
418static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
419static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
420static FNVMXEXITHANDLER hmR0VmxExitMwait;
421static FNVMXEXITHANDLER hmR0VmxExitMtf;
422static FNVMXEXITHANDLER hmR0VmxExitMonitor;
423static FNVMXEXITHANDLER hmR0VmxExitPause;
424static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
425static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
426static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
427static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
428static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
429static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
430static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
431static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
432static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
433static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
434static FNVMXEXITHANDLER hmR0VmxExitRdrand;
435static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
436/** @} */
437
438static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
439static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
440static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
441static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
442static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
443static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
444static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
445static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu);
446
447
448/*********************************************************************************************************************************
449* Global Variables *
450*********************************************************************************************************************************/
451#ifdef HMVMX_USE_FUNCTION_TABLE
452
453/**
454 * VMX_EXIT dispatch table.
455 */
456static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
457{
458 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
459 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
460 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
461 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
462 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
463 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
464 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
465 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
466 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
467 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
468 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
469 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
470 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
471 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
472 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
473 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
474 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
475 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
476 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
477#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
478 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
479 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
480 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
481 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
482 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
483 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
484 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
485 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
486 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
487#else
488 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
489 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
490 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
491 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
492 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
493 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
494 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
495 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
496 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
497#endif
498 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
499 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
500 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
501 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
502 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
503 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
504 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
505 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
506 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
507 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
508 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
509 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
510 /* 40 UNDEFINED */ hmR0VmxExitPause,
511 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
512 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
513 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
514 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
515 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
516 /* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitXdtrAccess,
517 /* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitXdtrAccess,
518 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
519 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
520 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
521 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
522 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
523 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
524 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
525 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
526 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
527 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
528 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
529 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
530 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
531 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
532 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
533 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
534 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
535};
536#endif /* HMVMX_USE_FUNCTION_TABLE */
537
538#ifdef VBOX_STRICT
539static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
540{
541 /* 0 */ "(Not Used)",
542 /* 1 */ "VMCALL executed in VMX root operation.",
543 /* 2 */ "VMCLEAR with invalid physical address.",
544 /* 3 */ "VMCLEAR with VMXON pointer.",
545 /* 4 */ "VMLAUNCH with non-clear VMCS.",
546 /* 5 */ "VMRESUME with non-launched VMCS.",
547 /* 6 */ "VMRESUME after VMXOFF",
548 /* 7 */ "VM-entry with invalid control fields.",
549 /* 8 */ "VM-entry with invalid host state fields.",
550 /* 9 */ "VMPTRLD with invalid physical address.",
551 /* 10 */ "VMPTRLD with VMXON pointer.",
552 /* 11 */ "VMPTRLD with incorrect revision identifier.",
553 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
554 /* 13 */ "VMWRITE to read-only VMCS component.",
555 /* 14 */ "(Not Used)",
556 /* 15 */ "VMXON executed in VMX root operation.",
557 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
558 /* 17 */ "VM-entry with non-launched executing VMCS.",
559 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
560 /* 19 */ "VMCALL with non-clear VMCS.",
561 /* 20 */ "VMCALL with invalid VM-exit control fields.",
562 /* 21 */ "(Not Used)",
563 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
564 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
565 /* 24 */ "VMCALL with invalid SMM-monitor features.",
566 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
567 /* 26 */ "VM-entry with events blocked by MOV SS.",
568 /* 27 */ "(Not Used)",
569 /* 28 */ "Invalid operand to INVEPT/INVVPID."
570};
571#endif /* VBOX_STRICT */
572
573
574/**
575 * Updates the VM's last error record.
576 *
577 * If there was a VMX instruction error, reads the error data from the VMCS and
578 * updates VCPU's last error record as well.
579 *
580 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
581 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
582 * VERR_VMX_INVALID_VMCS_FIELD.
583 * @param rc The error code.
584 */
585static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
586{
587 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
588 || rc == VERR_VMX_UNABLE_TO_START_VM)
589 {
590 AssertPtrReturnVoid(pVCpu);
591 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
592 }
593 pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
594}
595
596
597/**
598 * Reads the VM-entry interruption-information field from the VMCS into the VMX
599 * transient structure.
600 *
601 * @returns VBox status code.
602 * @param pVmxTransient Pointer to the VMX transient structure.
603 *
604 * @remarks No-long-jump zone!!!
605 */
606DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
607{
608 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
609 AssertRCReturn(rc, rc);
610 return VINF_SUCCESS;
611}
612
613#ifdef VBOX_STRICT
614/**
615 * Reads the VM-entry exception error code field from the VMCS into
616 * the VMX transient structure.
617 *
618 * @returns VBox status code.
619 * @param pVmxTransient Pointer to the VMX transient structure.
620 *
621 * @remarks No-long-jump zone!!!
622 */
623DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
624{
625 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
626 AssertRCReturn(rc, rc);
627 return VINF_SUCCESS;
628}
629
630
631/**
632 * Reads the VM-entry exception error code field from the VMCS into
633 * the VMX transient structure.
634 *
635 * @returns VBox status code.
636 * @param pVmxTransient Pointer to the VMX transient structure.
637 *
638 * @remarks No-long-jump zone!!!
639 */
640DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
641{
642 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
643 AssertRCReturn(rc, rc);
644 return VINF_SUCCESS;
645}
646#endif /* VBOX_STRICT */
647
648
649/**
650 * Reads the VM-exit interruption-information field from the VMCS into the VMX
651 * transient structure.
652 *
653 * @returns VBox status code.
654 * @param pVmxTransient Pointer to the VMX transient structure.
655 */
656DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
657{
658 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
659 {
660 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
661 AssertRCReturn(rc,rc);
662 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_INFO;
663 }
664 return VINF_SUCCESS;
665}
666
667
668/**
669 * Reads the VM-exit interruption error code from the VMCS into the VMX
670 * transient structure.
671 *
672 * @returns VBox status code.
673 * @param pVmxTransient Pointer to the VMX transient structure.
674 */
675DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
676{
677 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
678 {
679 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
680 AssertRCReturn(rc, rc);
681 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
682 }
683 return VINF_SUCCESS;
684}
685
686
687/**
688 * Reads the VM-exit instruction length field from the VMCS into the VMX
689 * transient structure.
690 *
691 * @returns VBox status code.
692 * @param pVmxTransient Pointer to the VMX transient structure.
693 */
694DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
695{
696 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
697 {
698 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
699 AssertRCReturn(rc, rc);
700 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_LEN;
701 }
702 return VINF_SUCCESS;
703}
704
705
706/**
707 * Reads the VM-exit instruction-information field from the VMCS into
708 * the VMX transient structure.
709 *
710 * @returns VBox status code.
711 * @param pVmxTransient Pointer to the VMX transient structure.
712 */
713DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
714{
715 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
716 {
717 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
718 AssertRCReturn(rc, rc);
719 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_INFO;
720 }
721 return VINF_SUCCESS;
722}
723
724
725/**
726 * Reads the exit code qualification from the VMCS into the VMX transient
727 * structure.
728 *
729 * @returns VBox status code.
730 * @param pVCpu The cross context virtual CPU structure of the
731 * calling EMT. (Required for the VMCS cache case.)
732 * @param pVmxTransient Pointer to the VMX transient structure.
733 */
734DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
735{
736 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
737 {
738 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
739 AssertRCReturn(rc, rc);
740 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION;
741 }
742 return VINF_SUCCESS;
743}
744
745
746/**
747 * Reads the IDT-vectoring information field from the VMCS into the VMX
748 * transient structure.
749 *
750 * @returns VBox status code.
751 * @param pVmxTransient Pointer to the VMX transient structure.
752 *
753 * @remarks No-long-jump zone!!!
754 */
755DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
756{
757 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
758 {
759 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
760 AssertRCReturn(rc, rc);
761 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_INFO;
762 }
763 return VINF_SUCCESS;
764}
765
766
767/**
768 * Reads the IDT-vectoring error code from the VMCS into the VMX
769 * transient structure.
770 *
771 * @returns VBox status code.
772 * @param pVmxTransient Pointer to the VMX transient structure.
773 */
774DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
775{
776 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
777 {
778 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
779 AssertRCReturn(rc, rc);
780 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
781 }
782 return VINF_SUCCESS;
783}
784
785
786/**
787 * Enters VMX root mode operation on the current CPU.
788 *
789 * @returns VBox status code.
790 * @param pVM The cross context VM structure. Can be
791 * NULL, after a resume.
792 * @param HCPhysCpuPage Physical address of the VMXON region.
793 * @param pvCpuPage Pointer to the VMXON region.
794 */
795static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
796{
797 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
798 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
799 Assert(pvCpuPage);
800 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
801
802 if (pVM)
803 {
804 /* Write the VMCS revision dword to the VMXON region. */
805 *(uint32_t *)pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
806 }
807
808 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
809 RTCCUINTREG fEFlags = ASMIntDisableFlags();
810
811 /* Enable the VMX bit in CR4 if necessary. */
812 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
813
814 /* Enter VMX root mode. */
815 int rc = VMXEnable(HCPhysCpuPage);
816 if (RT_FAILURE(rc))
817 {
818 if (!(uOldCr4 & X86_CR4_VMXE))
819 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
820
821 if (pVM)
822 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
823 }
824
825 /* Restore interrupts. */
826 ASMSetFlags(fEFlags);
827 return rc;
828}
829
830
831/**
832 * Exits VMX root mode operation on the current CPU.
833 *
834 * @returns VBox status code.
835 */
836static int hmR0VmxLeaveRootMode(void)
837{
838 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
839
840 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
841 RTCCUINTREG fEFlags = ASMIntDisableFlags();
842
843 /* If we're for some reason not in VMX root mode, then don't leave it. */
844 RTCCUINTREG uHostCR4 = ASMGetCR4();
845
846 int rc;
847 if (uHostCR4 & X86_CR4_VMXE)
848 {
849 /* Exit VMX root mode and clear the VMX bit in CR4. */
850 VMXDisable();
851 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
852 rc = VINF_SUCCESS;
853 }
854 else
855 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
856
857 /* Restore interrupts. */
858 ASMSetFlags(fEFlags);
859 return rc;
860}
861
862
863/**
864 * Allocates and maps one physically contiguous page. The allocated page is
865 * zero'd out. (Used by various VT-x structures).
866 *
867 * @returns IPRT status code.
868 * @param pMemObj Pointer to the ring-0 memory object.
869 * @param ppVirt Where to store the virtual address of the
870 * allocation.
871 * @param pHCPhys Where to store the physical address of the
872 * allocation.
873 */
874static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
875{
876 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
877 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
878 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
879
880 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
881 if (RT_FAILURE(rc))
882 return rc;
883 *ppVirt = RTR0MemObjAddress(*pMemObj);
884 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
885 ASMMemZero32(*ppVirt, PAGE_SIZE);
886 return VINF_SUCCESS;
887}
888
889
890/**
891 * Frees and unmaps an allocated physical page.
892 *
893 * @param pMemObj Pointer to the ring-0 memory object.
894 * @param ppVirt Where to re-initialize the virtual address of
895 * allocation as 0.
896 * @param pHCPhys Where to re-initialize the physical address of the
897 * allocation as 0.
898 */
899static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
900{
901 AssertPtr(pMemObj);
902 AssertPtr(ppVirt);
903 AssertPtr(pHCPhys);
904 if (*pMemObj != NIL_RTR0MEMOBJ)
905 {
906 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
907 AssertRC(rc);
908 *pMemObj = NIL_RTR0MEMOBJ;
909 *ppVirt = 0;
910 *pHCPhys = 0;
911 }
912}
913
914
915/**
916 * Worker function to free VT-x related structures.
917 *
918 * @returns IPRT status code.
919 * @param pVM The cross context VM structure.
920 */
921static void hmR0VmxStructsFree(PVM pVM)
922{
923 for (VMCPUID i = 0; i < pVM->cCpus; i++)
924 {
925 PVMCPU pVCpu = &pVM->aCpus[i];
926 AssertPtr(pVCpu);
927
928 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
929 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
930
931 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
932 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
933
934 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
935 }
936
937 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
938#ifdef VBOX_WITH_CRASHDUMP_MAGIC
939 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
940#endif
941}
942
943
944/**
945 * Worker function to allocate VT-x related VM structures.
946 *
947 * @returns IPRT status code.
948 * @param pVM The cross context VM structure.
949 */
950static int hmR0VmxStructsAlloc(PVM pVM)
951{
952 /*
953 * Initialize members up-front so we can cleanup properly on allocation failure.
954 */
955#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
956 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
957 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
958 pVM->hm.s.vmx.HCPhys##a_Name = 0;
959
960#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
961 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
962 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
963 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
964
965#ifdef VBOX_WITH_CRASHDUMP_MAGIC
966 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
967#endif
968 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
969
970 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
971 for (VMCPUID i = 0; i < pVM->cCpus; i++)
972 {
973 PVMCPU pVCpu = &pVM->aCpus[i];
974 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
975 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
976 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
977 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
978 }
979#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
980#undef VMXLOCAL_INIT_VM_MEMOBJ
981
982 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
983 AssertReturnStmt(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE) <= PAGE_SIZE,
984 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
985 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
986
987 /*
988 * Allocate all the VT-x structures.
989 */
990 int rc = VINF_SUCCESS;
991#ifdef VBOX_WITH_CRASHDUMP_MAGIC
992 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
993 if (RT_FAILURE(rc))
994 goto cleanup;
995 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
996 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
997#endif
998
999 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1000 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1001 {
1002 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1003 &pVM->hm.s.vmx.HCPhysApicAccess);
1004 if (RT_FAILURE(rc))
1005 goto cleanup;
1006 }
1007
1008 /*
1009 * Initialize per-VCPU VT-x structures.
1010 */
1011 for (VMCPUID i = 0; i < pVM->cCpus; i++)
1012 {
1013 PVMCPU pVCpu = &pVM->aCpus[i];
1014 AssertPtr(pVCpu);
1015
1016 /* Allocate the VM control structure (VMCS). */
1017 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1018 if (RT_FAILURE(rc))
1019 goto cleanup;
1020
1021 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1022 if ( PDMHasApic(pVM)
1023 && (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1024 {
1025 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1026 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1027 if (RT_FAILURE(rc))
1028 goto cleanup;
1029 }
1030
1031 /*
1032 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1033 * transparent accesses of specific MSRs.
1034 *
1035 * If the condition for enabling MSR bitmaps changes here, don't forget to
1036 * update HMAreMsrBitmapsAvailable().
1037 */
1038 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1039 {
1040 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1041 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1042 if (RT_FAILURE(rc))
1043 goto cleanup;
1044 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1045 }
1046
1047 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1048 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1049 if (RT_FAILURE(rc))
1050 goto cleanup;
1051
1052 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1053 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1054 if (RT_FAILURE(rc))
1055 goto cleanup;
1056 }
1057
1058 return VINF_SUCCESS;
1059
1060cleanup:
1061 hmR0VmxStructsFree(pVM);
1062 return rc;
1063}
1064
1065
1066/**
1067 * Does global VT-x initialization (called during module initialization).
1068 *
1069 * @returns VBox status code.
1070 */
1071VMMR0DECL(int) VMXR0GlobalInit(void)
1072{
1073#ifdef HMVMX_USE_FUNCTION_TABLE
1074 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1075# ifdef VBOX_STRICT
1076 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1077 Assert(g_apfnVMExitHandlers[i]);
1078# endif
1079#endif
1080 return VINF_SUCCESS;
1081}
1082
1083
1084/**
1085 * Does global VT-x termination (called during module termination).
1086 */
1087VMMR0DECL(void) VMXR0GlobalTerm()
1088{
1089 /* Nothing to do currently. */
1090}
1091
1092
1093/**
1094 * Sets up and activates VT-x on the current CPU.
1095 *
1096 * @returns VBox status code.
1097 * @param pHostCpu Pointer to the global CPU info struct.
1098 * @param pVM The cross context VM structure. Can be
1099 * NULL after a host resume operation.
1100 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1101 * fEnabledByHost is @c true).
1102 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1103 * @a fEnabledByHost is @c true).
1104 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1105 * enable VT-x on the host.
1106 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1107 */
1108VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1109 void *pvMsrs)
1110{
1111 Assert(pHostCpu);
1112 Assert(pvMsrs);
1113 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1114
1115 /* Enable VT-x if it's not already enabled by the host. */
1116 if (!fEnabledByHost)
1117 {
1118 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1119 if (RT_FAILURE(rc))
1120 return rc;
1121 }
1122
1123 /*
1124 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1125 * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1126 * invalidated when flushing by VPID.
1127 */
1128 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1129 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1130 {
1131 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
1132 pHostCpu->fFlushAsidBeforeUse = false;
1133 }
1134 else
1135 pHostCpu->fFlushAsidBeforeUse = true;
1136
1137 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1138 ++pHostCpu->cTlbFlushes;
1139
1140 return VINF_SUCCESS;
1141}
1142
1143
1144/**
1145 * Deactivates VT-x on the current CPU.
1146 *
1147 * @returns VBox status code.
1148 * @param pHostCpu Pointer to the global CPU info struct.
1149 * @param pvCpuPage Pointer to the VMXON region.
1150 * @param HCPhysCpuPage Physical address of the VMXON region.
1151 *
1152 * @remarks This function should never be called when SUPR0EnableVTx() or
1153 * similar was used to enable VT-x on the host.
1154 */
1155VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1156{
1157 RT_NOREF3(pHostCpu, pvCpuPage, HCPhysCpuPage);
1158
1159 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1160 return hmR0VmxLeaveRootMode();
1161}
1162
1163
1164/**
1165 * Sets the permission bits for the specified MSR in the MSR bitmap.
1166 *
1167 * @param pVCpu The cross context virtual CPU structure.
1168 * @param uMsr The MSR value.
1169 * @param enmRead Whether reading this MSR causes a VM-exit.
1170 * @param enmWrite Whether writing this MSR causes a VM-exit.
1171 */
1172static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1173{
1174 int32_t iBit;
1175 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1176
1177 /*
1178 * MSR Layout:
1179 * Byte index MSR range Interpreted as
1180 * 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
1181 * 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
1182 * 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
1183 * 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
1184 *
1185 * A bit corresponding to an MSR within the above range causes a VM-exit
1186 * if the bit is 1 on executions of RDMSR/WRMSR.
1187 *
1188 * If an MSR falls out of the MSR range, it always cause a VM-exit.
1189 *
1190 * See Intel spec. 24.6.9 "MSR-Bitmap Address".
1191 */
1192 if (uMsr <= 0x00001fff)
1193 iBit = uMsr;
1194 else if (uMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1195 {
1196 iBit = uMsr - UINT32_C(0xc0000000);
1197 pbMsrBitmap += 0x400;
1198 }
1199 else
1200 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1201
1202 Assert(iBit <= 0x1fff);
1203 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1204 ASMBitSet(pbMsrBitmap, iBit);
1205 else
1206 ASMBitClear(pbMsrBitmap, iBit);
1207
1208 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1209 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1210 else
1211 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1212}
1213
1214
1215/**
1216 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1217 * area.
1218 *
1219 * @returns VBox status code.
1220 * @param pVCpu The cross context virtual CPU structure.
1221 * @param cMsrs The number of MSRs.
1222 */
1223static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1224{
1225 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1226 uint64_t const uVmxMiscMsr = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc;
1227 uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(uVmxMiscMsr);
1228 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1229 {
1230 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1231 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1232 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1233 }
1234
1235 /* Update number of guest MSRs to load/store across the world-switch. */
1236 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1237 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1238
1239 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1240 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1241 AssertRCReturn(rc, rc);
1242
1243 /* Update the VCPU's copy of the MSR count. */
1244 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1245
1246 return VINF_SUCCESS;
1247}
1248
1249
1250/**
1251 * Adds a new (or updates the value of an existing) guest/host MSR
1252 * pair to be swapped during the world-switch as part of the
1253 * auto-load/store MSR area in the VMCS.
1254 *
1255 * @returns VBox status code.
1256 * @param pVCpu The cross context virtual CPU structure.
1257 * @param uMsr The MSR.
1258 * @param uGuestMsrValue Value of the guest MSR.
1259 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1260 * necessary.
1261 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1262 * its value was updated. Optional, can be NULL.
1263 */
1264static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1265 bool *pfAddedAndUpdated)
1266{
1267 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1268 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1269 uint32_t i;
1270 for (i = 0; i < cMsrs; i++)
1271 {
1272 if (pGuestMsr->u32Msr == uMsr)
1273 break;
1274 pGuestMsr++;
1275 }
1276
1277 bool fAdded = false;
1278 if (i == cMsrs)
1279 {
1280 ++cMsrs;
1281 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1282 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1283
1284 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1285 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1286 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1287
1288 fAdded = true;
1289 }
1290
1291 /* Update the MSR values in the auto-load/store MSR area. */
1292 pGuestMsr->u32Msr = uMsr;
1293 pGuestMsr->u64Value = uGuestMsrValue;
1294
1295 /* Create/update the MSR slot in the host MSR area. */
1296 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1297 pHostMsr += i;
1298 pHostMsr->u32Msr = uMsr;
1299
1300 /*
1301 * Update the host MSR only when requested by the caller AND when we're
1302 * adding it to the auto-load/store area. Otherwise, it would have been
1303 * updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1304 */
1305 bool fUpdatedMsrValue = false;
1306 if ( fAdded
1307 && fUpdateHostMsr)
1308 {
1309 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1310 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1311 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1312 fUpdatedMsrValue = true;
1313 }
1314
1315 if (pfAddedAndUpdated)
1316 *pfAddedAndUpdated = fUpdatedMsrValue;
1317 return VINF_SUCCESS;
1318}
1319
1320
1321/**
1322 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1323 * auto-load/store MSR area in the VMCS.
1324 *
1325 * @returns VBox status code.
1326 * @param pVCpu The cross context virtual CPU structure.
1327 * @param uMsr The MSR.
1328 */
1329static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1330{
1331 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1332 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1333 for (uint32_t i = 0; i < cMsrs; i++)
1334 {
1335 /* Find the MSR. */
1336 if (pGuestMsr->u32Msr == uMsr)
1337 {
1338 /* If it's the last MSR, simply reduce the count. */
1339 if (i == cMsrs - 1)
1340 {
1341 --cMsrs;
1342 break;
1343 }
1344
1345 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1346 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1347 pLastGuestMsr += cMsrs - 1;
1348 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1349 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1350
1351 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1352 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1353 pLastHostMsr += cMsrs - 1;
1354 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1355 pHostMsr->u64Value = pLastHostMsr->u64Value;
1356 --cMsrs;
1357 break;
1358 }
1359 pGuestMsr++;
1360 }
1361
1362 /* Update the VMCS if the count changed (meaning the MSR was found). */
1363 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1364 {
1365 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1366 AssertRCReturn(rc, rc);
1367
1368 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1369 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1370 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1371
1372 Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1373 return VINF_SUCCESS;
1374 }
1375
1376 return VERR_NOT_FOUND;
1377}
1378
1379
1380/**
1381 * Checks if the specified guest MSR is part of the auto-load/store area in
1382 * the VMCS.
1383 *
1384 * @returns true if found, false otherwise.
1385 * @param pVCpu The cross context virtual CPU structure.
1386 * @param uMsr The MSR to find.
1387 */
1388static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1389{
1390 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1391 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1392
1393 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1394 {
1395 if (pGuestMsr->u32Msr == uMsr)
1396 return true;
1397 }
1398 return false;
1399}
1400
1401
1402/**
1403 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1404 *
1405 * @param pVCpu The cross context virtual CPU structure.
1406 *
1407 * @remarks No-long-jump zone!!!
1408 */
1409static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1410{
1411 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1412 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1413 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1414 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1415
1416 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1417 {
1418 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1419
1420 /*
1421 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1422 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1423 */
1424 if (pHostMsr->u32Msr == MSR_K6_EFER)
1425 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1426 else
1427 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1428 }
1429
1430 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1431}
1432
1433
1434/**
1435 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1436 * perform lazy restoration of the host MSRs while leaving VT-x.
1437 *
1438 * @param pVCpu The cross context virtual CPU structure.
1439 *
1440 * @remarks No-long-jump zone!!!
1441 */
1442static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1443{
1444 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1445
1446 /*
1447 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1448 */
1449 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1450 {
1451 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1452#if HC_ARCH_BITS == 64
1453 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1454 {
1455 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1456 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1457 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1458 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1459 }
1460#endif
1461 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1462 }
1463}
1464
1465
1466/**
1467 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1468 * lazily while leaving VT-x.
1469 *
1470 * @returns true if it does, false otherwise.
1471 * @param pVCpu The cross context virtual CPU structure.
1472 * @param uMsr The MSR to check.
1473 */
1474static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1475{
1476 NOREF(pVCpu);
1477#if HC_ARCH_BITS == 64
1478 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1479 {
1480 switch (uMsr)
1481 {
1482 case MSR_K8_LSTAR:
1483 case MSR_K6_STAR:
1484 case MSR_K8_SF_MASK:
1485 case MSR_K8_KERNEL_GS_BASE:
1486 return true;
1487 }
1488 }
1489#else
1490 RT_NOREF(pVCpu, uMsr);
1491#endif
1492 return false;
1493}
1494
1495
1496/**
1497 * Loads a set of guests MSRs to allow read/passthru to the guest.
1498 *
1499 * The name of this function is slightly confusing. This function does NOT
1500 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1501 * common prefix for functions dealing with "lazy restoration" of the shared
1502 * MSRs.
1503 *
1504 * @param pVCpu The cross context virtual CPU structure.
1505 *
1506 * @remarks No-long-jump zone!!!
1507 */
1508static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1509{
1510 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1511 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1512
1513 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1514#if HC_ARCH_BITS == 64
1515 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1516 {
1517 /*
1518 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1519 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1520 * we can skip a few MSR writes.
1521 *
1522 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1523 * guest MSR values in the guest-CPU context might be different to what's currently
1524 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1525 * CPU, see @bugref{8728}.
1526 */
1527 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1528 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1529 && pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1530 && pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1531 && pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1532 && pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1533 {
1534#ifdef VBOX_STRICT
1535 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1536 Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1537 Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1538 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
1539#endif
1540 }
1541 else
1542 {
1543 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1544 ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
1545 ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
1546 ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
1547 }
1548 }
1549#endif
1550 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1551}
1552
1553
1554/**
1555 * Performs lazy restoration of the set of host MSRs if they were previously
1556 * loaded with guest MSR values.
1557 *
1558 * @param pVCpu The cross context virtual CPU structure.
1559 *
1560 * @remarks No-long-jump zone!!!
1561 * @remarks The guest MSRs should have been saved back into the guest-CPU
1562 * context by hmR0VmxImportGuestState()!!!
1563 */
1564static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1565{
1566 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1567 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1568
1569 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1570 {
1571 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1572#if HC_ARCH_BITS == 64
1573 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1574 {
1575 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1576 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1577 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1578 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1579 }
1580#endif
1581 }
1582 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1583}
1584
1585
1586/**
1587 * Verifies that our cached values of the VMCS fields are all consistent with
1588 * what's actually present in the VMCS.
1589 *
1590 * @returns VBox status code.
1591 * @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
1592 * @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
1593 * VMCS content. HMCPU error-field is
1594 * updated, see VMX_VCI_XXX.
1595 * @param pVCpu The cross context virtual CPU structure.
1596 */
1597static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1598{
1599 uint32_t u32Val;
1600 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1601 AssertRCReturn(rc, rc);
1602 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32EntryCtls == u32Val,
1603 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1604 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
1605 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1606
1607 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1608 AssertRCReturn(rc, rc);
1609 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ExitCtls == u32Val,
1610 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1611 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
1612 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1613
1614 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1615 AssertRCReturn(rc, rc);
1616 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32PinCtls == u32Val,
1617 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1618 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
1619 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1620
1621 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1622 AssertRCReturn(rc, rc);
1623 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls == u32Val,
1624 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1625 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
1626 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1627
1628 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
1629 {
1630 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1631 AssertRCReturn(rc, rc);
1632 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1633 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1634 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
1635 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1636 }
1637
1638 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
1639 AssertRCReturn(rc, rc);
1640 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32XcptBitmap == u32Val,
1641 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap, u32Val),
1642 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
1643 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1644
1645 uint64_t u64Val;
1646 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
1647 AssertRCReturn(rc, rc);
1648 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u64TscOffset == u64Val,
1649 ("Cache=%#RX64 VMCS=%#RX64\n", pVCpu->hm.s.vmx.u64TscOffset, u64Val),
1650 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
1651 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1652
1653 return VINF_SUCCESS;
1654}
1655
1656
1657#ifdef VBOX_STRICT
1658/**
1659 * Verifies that our cached host EFER value has not changed
1660 * since we cached it.
1661 *
1662 * @param pVCpu The cross context virtual CPU structure.
1663 */
1664static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1665{
1666 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1667
1668 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
1669 {
1670 uint64_t u64Val;
1671 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1672 AssertRC(rc);
1673
1674 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1675 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1676 }
1677}
1678
1679
1680/**
1681 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1682 * VMCS are correct.
1683 *
1684 * @param pVCpu The cross context virtual CPU structure.
1685 */
1686static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1687{
1688 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1689
1690 /* Verify MSR counts in the VMCS are what we think it should be. */
1691 uint32_t cMsrs;
1692 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1693 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1694
1695 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1696 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1697
1698 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1699 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1700
1701 PCVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1702 PCVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1703 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1704 {
1705 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1706 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1707 pGuestMsr->u32Msr, cMsrs));
1708
1709 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1710 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1711 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1712
1713 /* Verify that the permissions are as expected in the MSR bitmap. */
1714 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1715 {
1716 VMXMSREXITREAD enmRead;
1717 VMXMSREXITWRITE enmWrite;
1718 rc = HMVmxGetMsrPermission(pVCpu->hm.s.vmx.pvMsrBitmap, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1719 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("HMVmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1720 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1721 {
1722 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1723 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1724 }
1725 else
1726 {
1727 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1728 pGuestMsr->u32Msr, cMsrs));
1729 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1730 pGuestMsr->u32Msr, cMsrs));
1731 }
1732 }
1733 }
1734}
1735#endif /* VBOX_STRICT */
1736
1737
1738/**
1739 * Flushes the TLB using EPT.
1740 *
1741 * @returns VBox status code.
1742 * @param pVCpu The cross context virtual CPU structure of the calling
1743 * EMT. Can be NULL depending on @a enmTlbFlush.
1744 * @param enmTlbFlush Type of flush.
1745 *
1746 * @remarks Caller is responsible for making sure this function is called only
1747 * when NestedPaging is supported and providing @a enmTlbFlush that is
1748 * supported by the CPU.
1749 * @remarks Can be called with interrupts disabled.
1750 */
1751static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush)
1752{
1753 uint64_t au64Descriptor[2];
1754 if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
1755 au64Descriptor[0] = 0;
1756 else
1757 {
1758 Assert(pVCpu);
1759 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1760 }
1761 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1762
1763 int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
1764 AssertMsg(rc == VINF_SUCCESS,
1765 ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0, rc));
1766
1767 if ( RT_SUCCESS(rc)
1768 && pVCpu)
1769 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1770}
1771
1772
1773/**
1774 * Flushes the TLB using VPID.
1775 *
1776 * @returns VBox status code.
1777 * @param pVCpu The cross context virtual CPU structure of the calling
1778 * EMT. Can be NULL depending on @a enmTlbFlush.
1779 * @param enmTlbFlush Type of flush.
1780 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1781 * on @a enmTlbFlush).
1782 *
1783 * @remarks Can be called with interrupts disabled.
1784 */
1785static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
1786{
1787 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
1788
1789 uint64_t au64Descriptor[2];
1790 if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
1791 {
1792 au64Descriptor[0] = 0;
1793 au64Descriptor[1] = 0;
1794 }
1795 else
1796 {
1797 AssertPtr(pVCpu);
1798 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1799 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1800 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1801 au64Descriptor[1] = GCPtr;
1802 }
1803
1804 int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
1805 AssertMsg(rc == VINF_SUCCESS,
1806 ("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1807
1808 if ( RT_SUCCESS(rc)
1809 && pVCpu)
1810 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1811 NOREF(rc);
1812}
1813
1814
1815/**
1816 * Invalidates a guest page by guest virtual address. Only relevant for
1817 * EPT/VPID, otherwise there is nothing really to invalidate.
1818 *
1819 * @returns VBox status code.
1820 * @param pVCpu The cross context virtual CPU structure.
1821 * @param GCVirt Guest virtual address of the page to invalidate.
1822 */
1823VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
1824{
1825 AssertPtr(pVCpu);
1826 LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
1827
1828 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1829 if (!fFlushPending)
1830 {
1831 /*
1832 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1833 * the EPT case. See @bugref{6043} and @bugref{6177}.
1834 *
1835 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1836 * as this function maybe called in a loop with individual addresses.
1837 */
1838 PVM pVM = pVCpu->CTX_SUFF(pVM);
1839 if (pVM->hm.s.vmx.fVpid)
1840 {
1841 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1842
1843#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1844 /*
1845 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1846 * where executing INVVPID outside 64-bit mode does not flush translations of
1847 * 64-bit linear addresses, see @bugref{6208#c72}.
1848 */
1849 if (RT_HI_U32(GCVirt))
1850 fVpidFlush = false;
1851#endif
1852
1853 if (fVpidFlush)
1854 {
1855 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
1856 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1857 }
1858 else
1859 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1860 }
1861 else if (pVM->hm.s.fNestedPaging)
1862 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1863 }
1864
1865 return VINF_SUCCESS;
1866}
1867
1868
1869/**
1870 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1871 * case where neither EPT nor VPID is supported by the CPU.
1872 *
1873 * @param pVCpu The cross context virtual CPU structure.
1874 * @param pCpu Pointer to the global HM struct.
1875 *
1876 * @remarks Called with interrupts disabled.
1877 */
1878static void hmR0VmxFlushTaggedTlbNone(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1879{
1880 AssertPtr(pVCpu);
1881 AssertPtr(pCpu);
1882
1883 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1884
1885 Assert(pCpu->idCpu != NIL_RTCPUID);
1886 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1887 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1888 pVCpu->hm.s.fForceTLBFlush = false;
1889 return;
1890}
1891
1892
1893/**
1894 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1895 *
1896 * @param pVCpu The cross context virtual CPU structure.
1897 * @param pCpu Pointer to the global HM CPU struct.
1898 *
1899 * @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
1900 * nomenclature. The reason is, to avoid confusion in compare statements
1901 * since the host-CPU copies are named "ASID".
1902 *
1903 * @remarks Called with interrupts disabled.
1904 */
1905static void hmR0VmxFlushTaggedTlbBoth(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1906{
1907#ifdef VBOX_WITH_STATISTICS
1908 bool fTlbFlushed = false;
1909# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1910# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1911 if (!fTlbFlushed) \
1912 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1913 } while (0)
1914#else
1915# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1916# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1917#endif
1918
1919 AssertPtr(pCpu);
1920 AssertPtr(pVCpu);
1921 Assert(pCpu->idCpu != NIL_RTCPUID);
1922
1923 PVM pVM = pVCpu->CTX_SUFF(pVM);
1924 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1925 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1926 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1927
1928 /*
1929 * Force a TLB flush for the first world-switch if the current CPU differs from the one we
1930 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1931 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1932 * cannot reuse the current ASID anymore.
1933 */
1934 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1935 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1936 {
1937 ++pCpu->uCurrentAsid;
1938 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1939 {
1940 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1941 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1942 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1943 }
1944
1945 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1946 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1947 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1948
1949 /*
1950 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1951 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1952 */
1953 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1954 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1955 HMVMX_SET_TAGGED_TLB_FLUSHED();
1956 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1957 }
1958 else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
1959 {
1960 /*
1961 * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
1962 * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
1963 * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
1964 * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
1965 * mappings, see @bugref{6568}.
1966 *
1967 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
1968 */
1969 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1970 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1971 HMVMX_SET_TAGGED_TLB_FLUSHED();
1972 }
1973
1974 pVCpu->hm.s.fForceTLBFlush = false;
1975 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1976
1977 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1978 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1979 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1980 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1981 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1982 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
1983 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
1984 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1985 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1986
1987 /* Update VMCS with the VPID. */
1988 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
1989 AssertRC(rc);
1990
1991#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1992}
1993
1994
1995/**
1996 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
1997 *
1998 * @returns VBox status code.
1999 * @param pVCpu The cross context virtual CPU structure.
2000 * @param pCpu Pointer to the global HM CPU struct.
2001 *
2002 * @remarks Called with interrupts disabled.
2003 */
2004static void hmR0VmxFlushTaggedTlbEpt(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2005{
2006 AssertPtr(pVCpu);
2007 AssertPtr(pCpu);
2008 Assert(pCpu->idCpu != NIL_RTCPUID);
2009 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2010 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2011
2012 /*
2013 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2014 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2015 */
2016 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2017 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2018 {
2019 pVCpu->hm.s.fForceTLBFlush = true;
2020 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2021 }
2022
2023 /* Check for explicit TLB flushes. */
2024 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2025 {
2026 pVCpu->hm.s.fForceTLBFlush = true;
2027 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2028 }
2029
2030 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2031 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2032
2033 if (pVCpu->hm.s.fForceTLBFlush)
2034 {
2035 hmR0VmxFlushEpt(pVCpu, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2036 pVCpu->hm.s.fForceTLBFlush = false;
2037 }
2038}
2039
2040
2041/**
2042 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2043 *
2044 * @returns VBox status code.
2045 * @param pVCpu The cross context virtual CPU structure.
2046 * @param pCpu Pointer to the global HM CPU struct.
2047 *
2048 * @remarks Called with interrupts disabled.
2049 */
2050static void hmR0VmxFlushTaggedTlbVpid(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2051{
2052 AssertPtr(pVCpu);
2053 AssertPtr(pCpu);
2054 Assert(pCpu->idCpu != NIL_RTCPUID);
2055 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2056 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2057
2058 /*
2059 * Force a TLB flush for the first world switch if the current CPU differs from the one we
2060 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2061 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2062 * cannot reuse the current ASID anymore.
2063 */
2064 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2065 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2066 {
2067 pVCpu->hm.s.fForceTLBFlush = true;
2068 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2069 }
2070
2071 /* Check for explicit TLB flushes. */
2072 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2073 {
2074 /*
2075 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2076 * hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2077 * fExplicitFlush = true here and change the pCpu->fFlushAsidBeforeUse check below to
2078 * include fExplicitFlush's too) - an obscure corner case.
2079 */
2080 pVCpu->hm.s.fForceTLBFlush = true;
2081 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2082 }
2083
2084 PVM pVM = pVCpu->CTX_SUFF(pVM);
2085 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2086 if (pVCpu->hm.s.fForceTLBFlush)
2087 {
2088 ++pCpu->uCurrentAsid;
2089 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2090 {
2091 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2092 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2093 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2094 }
2095
2096 pVCpu->hm.s.fForceTLBFlush = false;
2097 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2098 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2099 if (pCpu->fFlushAsidBeforeUse)
2100 {
2101 if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2102 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2103 else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2104 {
2105 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2106 pCpu->fFlushAsidBeforeUse = false;
2107 }
2108 else
2109 {
2110 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2111 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2112 }
2113 }
2114 }
2115
2116 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2117 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2118 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2119 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2120 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2121 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2122 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2123
2124 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2125 AssertRC(rc);
2126}
2127
2128
2129/**
2130 * Flushes the guest TLB entry based on CPU capabilities.
2131 *
2132 * @param pVCpu The cross context virtual CPU structure.
2133 * @param pCpu Pointer to the global HM CPU struct.
2134 */
2135DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2136{
2137#ifdef HMVMX_ALWAYS_FLUSH_TLB
2138 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2139#endif
2140 PVM pVM = pVCpu->CTX_SUFF(pVM);
2141 switch (pVM->hm.s.vmx.enmTlbFlushType)
2142 {
2143 case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVCpu, pCpu); break;
2144 case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pVCpu, pCpu); break;
2145 case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pVCpu, pCpu); break;
2146 case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pVCpu, pCpu); break;
2147 default:
2148 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2149 break;
2150 }
2151 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2152}
2153
2154
2155/**
2156 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2157 * TLB entries from the host TLB before VM-entry.
2158 *
2159 * @returns VBox status code.
2160 * @param pVM The cross context VM structure.
2161 */
2162static int hmR0VmxSetupTaggedTlb(PVM pVM)
2163{
2164 /*
2165 * Determine optimal flush type for Nested Paging.
2166 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2167 * guest execution (see hmR3InitFinalizeR0()).
2168 */
2169 if (pVM->hm.s.fNestedPaging)
2170 {
2171 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2172 {
2173 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2174 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2175 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2176 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2177 else
2178 {
2179 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2180 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2181 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2182 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2183 }
2184
2185 /* Make sure the write-back cacheable memory type for EPT is supported. */
2186 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2187 {
2188 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2189 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2190 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2191 }
2192
2193 /* EPT requires a page-walk length of 4. */
2194 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2195 {
2196 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2197 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2198 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2199 }
2200 }
2201 else
2202 {
2203 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2204 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2205 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2206 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2207 }
2208 }
2209
2210 /*
2211 * Determine optimal flush type for VPID.
2212 */
2213 if (pVM->hm.s.vmx.fVpid)
2214 {
2215 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2216 {
2217 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2218 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2219 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2220 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2221 else
2222 {
2223 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2224 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2225 LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2226 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2227 LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2228 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2229 pVM->hm.s.vmx.fVpid = false;
2230 }
2231 }
2232 else
2233 {
2234 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2235 Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2236 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2237 pVM->hm.s.vmx.fVpid = false;
2238 }
2239 }
2240
2241 /*
2242 * Setup the handler for flushing tagged-TLBs.
2243 */
2244 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2245 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2246 else if (pVM->hm.s.fNestedPaging)
2247 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2248 else if (pVM->hm.s.vmx.fVpid)
2249 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2250 else
2251 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2252 return VINF_SUCCESS;
2253}
2254
2255
2256/**
2257 * Sets up pin-based VM-execution controls in the VMCS.
2258 *
2259 * @returns VBox status code.
2260 * @param pVCpu The cross context virtual CPU structure.
2261 *
2262 * @remarks We don't really care about optimizing vmwrites here as it's done only
2263 * once per VM and hence we don't care about VMCS-field cache comparisons.
2264 */
2265static int hmR0VmxSetupPinCtls(PVMCPU pVCpu)
2266{
2267 PVM pVM = pVCpu->CTX_SUFF(pVM);
2268 uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.disallowed0; /* Bits set here must always be set. */
2269 uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2270
2271 fVal |= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2272 | VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2273
2274 if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2275 fVal |= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2276
2277 /* Enable the VMX preemption timer. */
2278 if (pVM->hm.s.vmx.fUsePreemptTimer)
2279 {
2280 Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2281 fVal |= VMX_PIN_CTLS_PREEMPT_TIMER;
2282 }
2283
2284#if 0
2285 /* Enable posted-interrupt processing. */
2286 if (pVM->hm.s.fPostedIntrs)
2287 {
2288 Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2289 Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2290 fVal |= VMX_PIN_CTL_POSTED_INT;
2291 }
2292#endif
2293
2294 if ((fVal & fZap) != fVal)
2295 {
2296 LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2297 pVM->hm.s.vmx.Msrs.PinCtls.n.disallowed0, fVal, fZap));
2298 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2299 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2300 }
2301
2302 /* Commit it to the VMCS and update our cache. */
2303 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2304 AssertRCReturn(rc, rc);
2305 pVCpu->hm.s.vmx.u32PinCtls = fVal;
2306
2307 return VINF_SUCCESS;
2308}
2309
2310
2311/**
2312 * Sets up secondary processor-based VM-execution controls in the VMCS.
2313 *
2314 * @returns VBox status code.
2315 * @param pVCpu The cross context virtual CPU structure.
2316 *
2317 * @remarks We don't really care about optimizing vmwrites here as it's done only
2318 * once per VM and hence we don't care about VMCS-field cache comparisons.
2319 */
2320static int hmR0VmxSetupProcCtls2(PVMCPU pVCpu)
2321{
2322 PVM pVM = pVCpu->CTX_SUFF(pVM);
2323 uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2324 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2325
2326 /* WBINVD causes a VM-exit. */
2327 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2328 fVal |= VMX_PROC_CTLS2_WBINVD_EXIT;
2329
2330 /* Enable EPT (aka nested-paging). */
2331 if (pVM->hm.s.fNestedPaging)
2332 fVal |= VMX_PROC_CTLS2_EPT;
2333
2334 /*
2335 * Enable the INVPCID instruction if supported by the hardware and we expose
2336 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2337 */
2338 if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
2339 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2340 fVal |= VMX_PROC_CTLS2_INVPCID;
2341
2342 /* Enable VPID. */
2343 if (pVM->hm.s.vmx.fVpid)
2344 fVal |= VMX_PROC_CTLS2_VPID;
2345
2346 /* Enable Unrestricted guest execution. */
2347 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2348 fVal |= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
2349
2350#if 0
2351 if (pVM->hm.s.fVirtApicRegs)
2352 {
2353 /* Enable APIC-register virtualization. */
2354 Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
2355 fVal |= VMX_PROC_CTLS2_APIC_REG_VIRT;
2356
2357 /* Enable virtual-interrupt delivery. */
2358 Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
2359 fVal |= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
2360 }
2361#endif
2362
2363 /* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
2364 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2365 * done dynamically. */
2366 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
2367 {
2368 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2369 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2370 fVal |= VMX_PROC_CTLS2_VIRT_APIC_ACCESS; /* Virtualize APIC accesses. */
2371 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2372 AssertRCReturn(rc, rc);
2373 }
2374
2375 /* Enable RDTSCP. */
2376 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
2377 fVal |= VMX_PROC_CTLS2_RDTSCP;
2378
2379 /* Enable Pause-Loop exiting. */
2380 if ( pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT
2381 && pVM->hm.s.vmx.cPleGapTicks
2382 && pVM->hm.s.vmx.cPleWindowTicks)
2383 {
2384 fVal |= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
2385
2386 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2387 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2388 AssertRCReturn(rc, rc);
2389 }
2390
2391 if ((fVal & fZap) != fVal)
2392 {
2393 LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2394 pVM->hm.s.vmx.Msrs.ProcCtls2.n.disallowed0, fVal, fZap));
2395 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2396 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2397 }
2398
2399 /* Commit it to the VMCS and update our cache. */
2400 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2401 AssertRCReturn(rc, rc);
2402 pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2403
2404 return VINF_SUCCESS;
2405}
2406
2407
2408/**
2409 * Sets up processor-based VM-execution controls in the VMCS.
2410 *
2411 * @returns VBox status code.
2412 * @param pVCpu The cross context virtual CPU structure.
2413 *
2414 * @remarks We don't really care about optimizing vmwrites here as it's done only
2415 * once per VM and hence we don't care about VMCS-field cache comparisons.
2416 */
2417static int hmR0VmxSetupProcCtls(PVMCPU pVCpu)
2418{
2419 PVM pVM = pVCpu->CTX_SUFF(pVM);
2420 uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2421 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2422
2423 fVal |= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
2424 | VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2425 | VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2426 | VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2427 | VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2428 | VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2429 | VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2430
2431 /* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2432 if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
2433 || (pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
2434 {
2435 LogRelFunc(("Unsupported VMX_PROC_CTLS_MOV_DR_EXIT combo!"));
2436 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2437 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2438 }
2439
2440 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2441 if (!pVM->hm.s.fNestedPaging)
2442 {
2443 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2444 fVal |= VMX_PROC_CTLS_INVLPG_EXIT
2445 | VMX_PROC_CTLS_CR3_LOAD_EXIT
2446 | VMX_PROC_CTLS_CR3_STORE_EXIT;
2447 }
2448
2449 /* Use TPR shadowing if supported by the CPU. */
2450 if ( PDMHasApic(pVM)
2451 && pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
2452 {
2453 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2454 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2455 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2456 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2457 AssertRCReturn(rc, rc);
2458
2459 fVal |= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2460 /* CR8 writes cause a VM-exit based on TPR threshold. */
2461 Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
2462 Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
2463 }
2464 else
2465 {
2466 /*
2467 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2468 * Set this control only for 64-bit guests.
2469 */
2470 if (pVM->hm.s.fAllow64BitGuests)
2471 {
2472 fVal |= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2473 | VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2474 }
2475 }
2476
2477 /* Use MSR-bitmaps if supported by the CPU. */
2478 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2479 {
2480 fVal |= VMX_PROC_CTLS_USE_MSR_BITMAPS;
2481
2482 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2483 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2484 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2485 AssertRCReturn(rc, rc);
2486
2487 /*
2488 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2489 * automatically using dedicated fields in the VMCS.
2490 */
2491 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2492 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2493 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2494 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2495 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2496#if HC_ARCH_BITS == 64
2497 /*
2498 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2499 */
2500 if (pVM->hm.s.fAllow64BitGuests)
2501 {
2502 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2503 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2504 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2505 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2506 }
2507#endif
2508 /*
2509 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2510 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2511 */
2512 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2513 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2514
2515 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2516 }
2517
2518 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2519 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2520 fVal |= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
2521
2522 if ((fVal & fZap) != fVal)
2523 {
2524 LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2525 pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0, fVal, fZap));
2526 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2527 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2528 }
2529
2530 /* Commit it to the VMCS and update our cache. */
2531 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2532 AssertRCReturn(rc, rc);
2533 pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2534
2535 /* Set up secondary processor-based VM-execution controls if the CPU supports it. */
2536 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2537 return hmR0VmxSetupProcCtls2(pVCpu);
2538
2539 /* Sanity check, should not really happen. */
2540 if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2541 {
2542 LogRelFunc(("Unrestricted Guest enabled when secondary processor-based VM-execution controls not available\n"));
2543 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2544 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2545 }
2546
2547 /* Old CPUs without secondary processor-based VM-execution controls would end up here. */
2548 return VINF_SUCCESS;
2549}
2550
2551
2552/**
2553 * Sets up miscellaneous (everything other than Pin & Processor-based
2554 * VM-execution) control fields in the VMCS.
2555 *
2556 * @returns VBox status code.
2557 * @param pVCpu The cross context virtual CPU structure.
2558 */
2559static int hmR0VmxSetupMiscCtls(PVMCPU pVCpu)
2560{
2561 AssertPtr(pVCpu);
2562
2563 int rc = VERR_GENERAL_FAILURE;
2564
2565 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2566#if 0
2567 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2568 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2569 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2570
2571 /*
2572 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2573 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2574 * We thus use the exception bitmap to control it rather than use both.
2575 */
2576 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2577 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2578
2579 /* All IO & IOIO instructions cause VM-exits. */
2580 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2581 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2582
2583 /* Initialize the MSR-bitmap area. */
2584 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2585 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2586 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2587 AssertRCReturn(rc, rc);
2588#endif
2589
2590 /* Setup MSR auto-load/store area. */
2591 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2592 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2593 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2594 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2595 AssertRCReturn(rc, rc);
2596
2597 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2598 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2599 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2600 AssertRCReturn(rc, rc);
2601
2602 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2603 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2604 AssertRCReturn(rc, rc);
2605
2606 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2607#if 0
2608 /* Setup debug controls */
2609 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2610 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2611 AssertRCReturn(rc, rc);
2612#endif
2613
2614 return rc;
2615}
2616
2617
2618/**
2619 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2620 *
2621 * We shall setup those exception intercepts that don't change during the
2622 * lifetime of the VM here. The rest are done dynamically while loading the
2623 * guest state.
2624 *
2625 * @returns VBox status code.
2626 * @param pVCpu The cross context virtual CPU structure.
2627 */
2628static int hmR0VmxInitXcptBitmap(PVMCPU pVCpu)
2629{
2630 AssertPtr(pVCpu);
2631
2632 uint32_t uXcptBitmap;
2633
2634 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2635 uXcptBitmap = RT_BIT_32(X86_XCPT_AC);
2636
2637 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2638 and writes, and because recursive #DBs can cause the CPU hang, we must always
2639 intercept #DB. */
2640 uXcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2641
2642 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2643 if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
2644 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
2645
2646 /* Commit it to the VMCS. */
2647 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2648 AssertRCReturn(rc, rc);
2649
2650 /* Update our cache of the exception bitmap. */
2651 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
2652 return VINF_SUCCESS;
2653}
2654
2655
2656/**
2657 * Does per-VM VT-x initialization.
2658 *
2659 * @returns VBox status code.
2660 * @param pVM The cross context VM structure.
2661 */
2662VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2663{
2664 LogFlowFunc(("pVM=%p\n", pVM));
2665
2666 int rc = hmR0VmxStructsAlloc(pVM);
2667 if (RT_FAILURE(rc))
2668 {
2669 LogRelFunc(("hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2670 return rc;
2671 }
2672
2673 return VINF_SUCCESS;
2674}
2675
2676
2677/**
2678 * Does per-VM VT-x termination.
2679 *
2680 * @returns VBox status code.
2681 * @param pVM The cross context VM structure.
2682 */
2683VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2684{
2685 LogFlowFunc(("pVM=%p\n", pVM));
2686
2687#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2688 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2689 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2690#endif
2691 hmR0VmxStructsFree(pVM);
2692 return VINF_SUCCESS;
2693}
2694
2695
2696/**
2697 * Sets up the VM for execution under VT-x.
2698 * This function is only called once per-VM during initialization.
2699 *
2700 * @returns VBox status code.
2701 * @param pVM The cross context VM structure.
2702 */
2703VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2704{
2705 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2706 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2707
2708 LogFlowFunc(("pVM=%p\n", pVM));
2709
2710 /*
2711 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be
2712 * allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2713 * pRealModeTSS, see hmR3InitFinalizeR0Intel().
2714 */
2715 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2716 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2717 || !pVM->hm.s.vmx.pRealModeTSS))
2718 {
2719 LogRelFunc(("Invalid real-on-v86 state.\n"));
2720 return VERR_INTERNAL_ERROR;
2721 }
2722
2723 /* Initialize these always, see hmR3InitFinalizeR0().*/
2724 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
2725 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
2726
2727 /* Setup the tagged-TLB flush handlers. */
2728 int rc = hmR0VmxSetupTaggedTlb(pVM);
2729 if (RT_FAILURE(rc))
2730 {
2731 LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2732 return rc;
2733 }
2734
2735 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2736 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2737#if HC_ARCH_BITS == 64
2738 if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
2739 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2740 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
2741 {
2742 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2743 }
2744#endif
2745
2746 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2747 RTCCUINTREG const uHostCR4 = ASMGetCR4();
2748 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2749 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2750
2751 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2752 {
2753 PVMCPU pVCpu = &pVM->aCpus[i];
2754 AssertPtr(pVCpu);
2755 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2756
2757 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2758 Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2759
2760 /* Set revision dword at the beginning of the VMCS structure. */
2761 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
2762
2763 /* Set the VMCS launch state to "clear", see Intel spec. 31.6 "Preparation and launch a virtual machine". */
2764 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2765 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc\n", rc),
2766 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2767
2768 /* Load this VMCS as the current VMCS. */
2769 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2770 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc\n", rc),
2771 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2772
2773 rc = hmR0VmxSetupPinCtls(pVCpu);
2774 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc\n", rc),
2775 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2776
2777 rc = hmR0VmxSetupProcCtls(pVCpu);
2778 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc\n", rc),
2779 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2780
2781 rc = hmR0VmxSetupMiscCtls(pVCpu);
2782 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc\n", rc),
2783 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2784
2785 rc = hmR0VmxInitXcptBitmap(pVCpu);
2786 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc\n", rc),
2787 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2788
2789#if HC_ARCH_BITS == 32
2790 rc = hmR0VmxInitVmcsReadCache(pVCpu);
2791 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc\n", rc),
2792 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2793#endif
2794
2795 /* Sync any CPU internal VMCS data back into our VMCS in memory. */
2796 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2797 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc\n", rc),
2798 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2799
2800 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2801
2802 hmR0VmxUpdateErrorRecord(pVCpu, rc);
2803 }
2804
2805 return VINF_SUCCESS;
2806}
2807
2808
2809/**
2810 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2811 * the VMCS.
2812 *
2813 * @returns VBox status code.
2814 */
2815static int hmR0VmxExportHostControlRegs(void)
2816{
2817 RTCCUINTREG uReg = ASMGetCR0();
2818 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2819 AssertRCReturn(rc, rc);
2820
2821 uReg = ASMGetCR3();
2822 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2823 AssertRCReturn(rc, rc);
2824
2825 uReg = ASMGetCR4();
2826 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2827 AssertRCReturn(rc, rc);
2828 return rc;
2829}
2830
2831
2832/**
2833 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2834 * the host-state area in the VMCS.
2835 *
2836 * @returns VBox status code.
2837 * @param pVCpu The cross context virtual CPU structure.
2838 */
2839static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2840{
2841#if HC_ARCH_BITS == 64
2842/**
2843 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2844 * requirements. See hmR0VmxExportHostSegmentRegs().
2845 */
2846# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2847 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2848 { \
2849 bool fValidSelector = true; \
2850 if ((selValue) & X86_SEL_LDT) \
2851 { \
2852 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2853 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2854 } \
2855 if (fValidSelector) \
2856 { \
2857 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2858 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2859 } \
2860 (selValue) = 0; \
2861 }
2862
2863 /*
2864 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2865 * should -not- save the messed up state without restoring the original host-state,
2866 * see @bugref{7240}.
2867 *
2868 * This apparently can happen (most likely the FPU changes), deal with it rather than
2869 * asserting. Was observed booting Solaris 10u10 32-bit guest.
2870 */
2871 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2872 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2873 {
2874 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2875 pVCpu->idCpu));
2876 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2877 }
2878 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2879#else
2880 RT_NOREF(pVCpu);
2881#endif
2882
2883 /*
2884 * Host DS, ES, FS and GS segment registers.
2885 */
2886#if HC_ARCH_BITS == 64
2887 RTSEL uSelDS = ASMGetDS();
2888 RTSEL uSelES = ASMGetES();
2889 RTSEL uSelFS = ASMGetFS();
2890 RTSEL uSelGS = ASMGetGS();
2891#else
2892 RTSEL uSelDS = 0;
2893 RTSEL uSelES = 0;
2894 RTSEL uSelFS = 0;
2895 RTSEL uSelGS = 0;
2896#endif
2897
2898 /*
2899 * Host CS and SS segment registers.
2900 */
2901 RTSEL uSelCS = ASMGetCS();
2902 RTSEL uSelSS = ASMGetSS();
2903
2904 /*
2905 * Host TR segment register.
2906 */
2907 RTSEL uSelTR = ASMGetTR();
2908
2909#if HC_ARCH_BITS == 64
2910 /*
2911 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2912 * gain VM-entry and restore them before we get preempted.
2913 *
2914 * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2915 */
2916 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2917 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2918 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2919 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2920# undef VMXLOCAL_ADJUST_HOST_SEG
2921#endif
2922
2923 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2924 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2925 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2926 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2927 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2928 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2929 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2930 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2931 Assert(uSelCS);
2932 Assert(uSelTR);
2933
2934 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2935#if 0
2936 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE))
2937 Assert(uSelSS != 0);
2938#endif
2939
2940 /* Write these host selector fields into the host-state area in the VMCS. */
2941 int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2942 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2943#if HC_ARCH_BITS == 64
2944 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2945 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2946 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2947 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2948#else
2949 NOREF(uSelDS);
2950 NOREF(uSelES);
2951 NOREF(uSelFS);
2952 NOREF(uSelGS);
2953#endif
2954 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2955 AssertRCReturn(rc, rc);
2956
2957 /*
2958 * Host GDTR and IDTR.
2959 */
2960 RTGDTR Gdtr;
2961 RTIDTR Idtr;
2962 RT_ZERO(Gdtr);
2963 RT_ZERO(Idtr);
2964 ASMGetGDTR(&Gdtr);
2965 ASMGetIDTR(&Idtr);
2966 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2967 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2968 AssertRCReturn(rc, rc);
2969
2970#if HC_ARCH_BITS == 64
2971 /*
2972 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
2973 * them to the maximum limit (0xffff) on every VM-exit.
2974 */
2975 if (Gdtr.cbGdt != 0xffff)
2976 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2977
2978 /*
2979 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2980 * Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
2981 * as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
2982 * However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
2983 * uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
2984 * alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
2985 * at 0xffff on hosts where we are sure it won't cause trouble.
2986 */
2987# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
2988 if (Idtr.cbIdt < 0x0fff)
2989# else
2990 if (Idtr.cbIdt != 0xffff)
2991# endif
2992 {
2993 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
2994 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2995 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2996 }
2997#endif
2998
2999 /*
3000 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3001 * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3002 * RPL should be too in most cases.
3003 */
3004 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3005 ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3006
3007 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3008#if HC_ARCH_BITS == 64
3009 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3010
3011 /*
3012 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3013 * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3014 * restoration if the host has something else. Task switching is not supported in 64-bit
3015 * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3016 * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3017 *
3018 * [1] See Intel spec. 3.5 "System Descriptor Types".
3019 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3020 */
3021 PVM pVM = pVCpu->CTX_SUFF(pVM);
3022 Assert(pDesc->System.u4Type == 11);
3023 if ( pDesc->System.u16LimitLow != 0x67
3024 || pDesc->System.u4LimitHigh)
3025 {
3026 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3027 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3028 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3029 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3030 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3031 }
3032
3033 /*
3034 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3035 */
3036 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3037 {
3038 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3039 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3040 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3041 {
3042 /* The GDT is read-only but the writable GDT is available. */
3043 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3044 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3045 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3046 AssertRCReturn(rc, rc);
3047 }
3048 }
3049#else
3050 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3051#endif
3052 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3053 AssertRCReturn(rc, rc);
3054
3055 /*
3056 * Host FS base and GS base.
3057 */
3058#if HC_ARCH_BITS == 64
3059 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3060 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3061 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3062 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3063 AssertRCReturn(rc, rc);
3064
3065 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3066 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3067 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3068 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3069 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3070#endif
3071 return VINF_SUCCESS;
3072}
3073
3074
3075/**
3076 * Exports certain host MSRs in the VM-exit MSR-load area and some in the
3077 * host-state area of the VMCS.
3078 *
3079 * Theses MSRs will be automatically restored on the host after every successful
3080 * VM-exit.
3081 *
3082 * @returns VBox status code.
3083 * @param pVCpu The cross context virtual CPU structure.
3084 *
3085 * @remarks No-long-jump zone!!!
3086 */
3087static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3088{
3089 AssertPtr(pVCpu);
3090 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3091
3092 /*
3093 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3094 * rather than swapping them on every VM-entry.
3095 */
3096 hmR0VmxLazySaveHostMsrs(pVCpu);
3097
3098 /*
3099 * Host Sysenter MSRs.
3100 */
3101 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3102#if HC_ARCH_BITS == 32
3103 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3104 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3105#else
3106 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3107 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3108#endif
3109 AssertRCReturn(rc, rc);
3110
3111 /*
3112 * Host EFER MSR.
3113 *
3114 * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3115 * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3116 */
3117 PVM pVM = pVCpu->CTX_SUFF(pVM);
3118 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3119 {
3120 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3121 AssertRCReturn(rc, rc);
3122 }
3123
3124 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3125
3126 return VINF_SUCCESS;
3127}
3128
3129
3130/**
3131 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3132 *
3133 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3134 * these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3135 * hmR0VMxExportGuestEntryCtls().
3136 *
3137 * @returns true if we need to load guest EFER, false otherwise.
3138 * @param pVCpu The cross context virtual CPU structure.
3139 *
3140 * @remarks Requires EFER, CR4.
3141 * @remarks No-long-jump zone!!!
3142 */
3143static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu)
3144{
3145#ifdef HMVMX_ALWAYS_SWAP_EFER
3146 RT_NOREF(pVCpu);
3147 return true;
3148#else
3149
3150 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3151#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3152 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3153 if (CPUMIsGuestInLongModeEx(pCtx))
3154 return false;
3155#endif
3156
3157 PVM pVM = pVCpu->CTX_SUFF(pVM);
3158 uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3159 uint64_t const u64GuestEfer = pCtx->msrEFER;
3160
3161 /*
3162 * For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3163 * guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3164 */
3165 if ( CPUMIsGuestInLongModeEx(pCtx)
3166 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3167 {
3168 return true;
3169 }
3170
3171 /*
3172 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3173 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3174 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3175 */
3176 if ( (pCtx->cr4 & X86_CR4_PAE)
3177 && (pCtx->cr0 & X86_CR0_PG)
3178 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3179 {
3180 /* Assert that host is NX capable. */
3181 Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
3182 return true;
3183 }
3184
3185 return false;
3186#endif
3187}
3188
3189
3190/**
3191 * Exports the guest state with appropriate VM-entry controls in the VMCS.
3192 *
3193 * These controls can affect things done on VM-exit; e.g. "load debug controls",
3194 * see Intel spec. 24.8.1 "VM-entry controls".
3195 *
3196 * @returns VBox status code.
3197 * @param pVCpu The cross context virtual CPU structure.
3198 *
3199 * @remarks Requires EFER.
3200 * @remarks No-long-jump zone!!!
3201 */
3202static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu)
3203{
3204 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3205 {
3206 PVM pVM = pVCpu->CTX_SUFF(pVM);
3207 uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
3208 uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3209
3210 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3211 fVal |= VMX_ENTRY_CTLS_LOAD_DEBUG;
3212
3213 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3214 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
3215 {
3216 fVal |= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
3217 Log4Func(("VMX_ENTRY_CTLS_IA32E_MODE_GUEST\n"));
3218 }
3219 else
3220 Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
3221
3222 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3223 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3224 && hmR0VmxShouldSwapEferMsr(pVCpu))
3225 {
3226 fVal |= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
3227 Log4Func(("VMX_ENTRY_CTLS_LOAD_EFER_MSR\n"));
3228 }
3229
3230 /*
3231 * The following should -not- be set (since we're not in SMM mode):
3232 * - VMX_ENTRY_CTLS_ENTRY_TO_SMM
3233 * - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
3234 */
3235
3236 /** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
3237 * VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
3238
3239 if ((fVal & fZap) != fVal)
3240 {
3241 Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3242 pVM->hm.s.vmx.Msrs.EntryCtls.n.disallowed0, fVal, fZap));
3243 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3244 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3245 }
3246
3247 /* Commit it to the VMCS and update our cache. */
3248 if (pVCpu->hm.s.vmx.u32EntryCtls != fVal)
3249 {
3250 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3251 AssertRCReturn(rc, rc);
3252 pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3253 }
3254
3255 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3256 }
3257 return VINF_SUCCESS;
3258}
3259
3260
3261/**
3262 * Exports the guest state with appropriate VM-exit controls in the VMCS.
3263 *
3264 * @returns VBox status code.
3265 * @param pVCpu The cross context virtual CPU structure.
3266 *
3267 * @remarks Requires EFER.
3268 */
3269static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu)
3270{
3271 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3272 {
3273 PVM pVM = pVCpu->CTX_SUFF(pVM);
3274 uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
3275 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3276
3277 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3278 fVal |= VMX_EXIT_CTLS_SAVE_DEBUG;
3279
3280 /*
3281 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3282 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3283 * hmR0VmxExportHostMsrs().
3284 */
3285#if HC_ARCH_BITS == 64
3286 fVal |= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
3287 Log4Func(("VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE\n"));
3288#else
3289 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3290 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3291 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3292 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3293 {
3294 /* The switcher returns to long mode, EFER is managed by the switcher. */
3295 fVal |= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
3296 Log4Func(("VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE\n"));
3297 }
3298 else
3299 Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
3300#endif
3301
3302 /* If the newer VMCS fields for managing EFER exists, use it. */
3303 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3304 && hmR0VmxShouldSwapEferMsr(pVCpu))
3305 {
3306 fVal |= VMX_EXIT_CTLS_SAVE_EFER_MSR
3307 | VMX_EXIT_CTLS_LOAD_EFER_MSR;
3308 Log4Func(("VMX_EXIT_CTLS_SAVE_EFER_MSR and VMX_EXIT_CTLS_LOAD_EFER_MSR\n"));
3309 }
3310
3311 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3312 Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
3313
3314 /** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
3315 * VMX_EXIT_CTLS_SAVE_PAT_MSR,
3316 * VMX_EXIT_CTLS_LOAD_PAT_MSR. */
3317
3318 /* Enable saving of the VMX preemption timer value on VM-exit. */
3319 if ( pVM->hm.s.vmx.fUsePreemptTimer
3320 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
3321 fVal |= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
3322
3323 if ((fVal & fZap) != fVal)
3324 {
3325 LogRelFunc(("Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3326 pVM->hm.s.vmx.Msrs.ExitCtls.n.disallowed0, fVal, fZap));
3327 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3328 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3329 }
3330
3331 /* Commit it to the VMCS and update our cache. */
3332 if (pVCpu->hm.s.vmx.u32ExitCtls != fVal)
3333 {
3334 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3335 AssertRCReturn(rc, rc);
3336 pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3337 }
3338
3339 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3340 }
3341 return VINF_SUCCESS;
3342}
3343
3344
3345/**
3346 * Sets the TPR threshold in the VMCS.
3347 *
3348 * @returns VBox status code.
3349 * @param pVCpu The cross context virtual CPU structure.
3350 * @param u32TprThreshold The TPR threshold (task-priority class only).
3351 */
3352DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3353{
3354 Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
3355 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3356 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3357}
3358
3359
3360/**
3361 * Exports the guest APIC TPR state into the VMCS.
3362 *
3363 * @returns VBox status code.
3364 * @param pVCpu The cross context virtual CPU structure.
3365 *
3366 * @remarks No-long-jump zone!!!
3367 */
3368static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3369{
3370 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3371 {
3372 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
3373
3374 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3375 && APICIsEnabled(pVCpu))
3376 {
3377 /*
3378 * Setup TPR shadowing.
3379 */
3380 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
3381 {
3382 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3383
3384 bool fPendingIntr = false;
3385 uint8_t u8Tpr = 0;
3386 uint8_t u8PendingIntr = 0;
3387 int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3388 AssertRCReturn(rc, rc);
3389
3390 /*
3391 * If there are interrupts pending but masked by the TPR, instruct VT-x to
3392 * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3393 * priority of the pending interrupt so we can deliver the interrupt. If there
3394 * are no interrupts pending, set threshold to 0 to not cause any
3395 * TPR-below-threshold VM-exits.
3396 */
3397 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3398 uint32_t u32TprThreshold = 0;
3399 if (fPendingIntr)
3400 {
3401 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3402 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3403 const uint8_t u8TprPriority = u8Tpr >> 4;
3404 if (u8PendingPriority <= u8TprPriority)
3405 u32TprThreshold = u8PendingPriority;
3406 }
3407
3408 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3409 AssertRCReturn(rc, rc);
3410 }
3411 }
3412 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3413 }
3414 return VINF_SUCCESS;
3415}
3416
3417
3418/**
3419 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3420 *
3421 * @returns Guest's interruptibility-state.
3422 * @param pVCpu The cross context virtual CPU structure.
3423 *
3424 * @remarks No-long-jump zone!!!
3425 */
3426static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu)
3427{
3428 /*
3429 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3430 */
3431 uint32_t fIntrState = 0;
3432 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3433 {
3434 /* If inhibition is active, RIP & RFLAGS should've been accessed
3435 (i.e. read previously from the VMCS or from ring-3). */
3436 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3437#ifdef VBOX_STRICT
3438 uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
3439 AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3440#endif
3441 if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3442 {
3443 if (pCtx->eflags.Bits.u1IF)
3444 fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
3445 else
3446 fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
3447 }
3448 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3449 {
3450 /*
3451 * We can clear the inhibit force flag as even if we go back to the recompiler
3452 * without executing guest code in VT-x, the flag's condition to be cleared is
3453 * met and thus the cleared state is correct.
3454 */
3455 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3456 }
3457 }
3458
3459 /*
3460 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3461 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3462 * setting this would block host-NMIs and IRET will not clear the blocking.
3463 *
3464 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3465 */
3466 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3467 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
3468 {
3469 fIntrState |= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
3470 }
3471
3472 return fIntrState;
3473}
3474
3475
3476/**
3477 * Exports the guest's interruptibility-state into the guest-state area in the
3478 * VMCS.
3479 *
3480 * @returns VBox status code.
3481 * @param pVCpu The cross context virtual CPU structure.
3482 * @param fIntrState The interruptibility-state to set.
3483 */
3484static int hmR0VmxExportGuestIntrState(PVMCPU pVCpu, uint32_t fIntrState)
3485{
3486 NOREF(pVCpu);
3487 AssertMsg(!(fIntrState & 0xfffffff0), ("%#x\n", fIntrState)); /* Bits 31:4 MBZ. */
3488 Assert((fIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3489 return VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
3490}
3491
3492
3493/**
3494 * Exports the exception intercepts required for guest execution in the VMCS.
3495 *
3496 * @returns VBox status code.
3497 * @param pVCpu The cross context virtual CPU structure.
3498 *
3499 * @remarks No-long-jump zone!!!
3500 */
3501static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3502{
3503 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3504 {
3505 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3506
3507 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3508 if (pVCpu->hm.s.fGIMTrapXcptUD)
3509 uXcptBitmap |= RT_BIT(X86_XCPT_UD);
3510#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3511 else
3512 uXcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3513#endif
3514
3515 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_AC));
3516 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_DB));
3517
3518 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3519 {
3520 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3521 AssertRCReturn(rc, rc);
3522 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3523 }
3524
3525 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3526 Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", uXcptBitmap));
3527 }
3528 return VINF_SUCCESS;
3529}
3530
3531
3532/**
3533 * Exports the guest's RIP into the guest-state area in the VMCS.
3534 *
3535 * @returns VBox status code.
3536 * @param pVCpu The cross context virtual CPU structure.
3537 *
3538 * @remarks No-long-jump zone!!!
3539 */
3540static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
3541{
3542 int rc = VINF_SUCCESS;
3543 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3544 {
3545 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
3546
3547 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
3548 AssertRCReturn(rc, rc);
3549
3550 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3551 Log4Func(("RIP=%#RX64\n", pVCpu->cpum.GstCtx.rip));
3552 }
3553 return rc;
3554}
3555
3556
3557/**
3558 * Exports the guest's RSP into the guest-state area in the VMCS.
3559 *
3560 * @returns VBox status code.
3561 * @param pVCpu The cross context virtual CPU structure.
3562 *
3563 * @remarks No-long-jump zone!!!
3564 */
3565static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
3566{
3567 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3568 {
3569 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
3570
3571 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
3572 AssertRCReturn(rc, rc);
3573
3574 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3575 }
3576 return VINF_SUCCESS;
3577}
3578
3579
3580/**
3581 * Exports the guest's RFLAGS into the guest-state area in the VMCS.
3582 *
3583 * @returns VBox status code.
3584 * @param pVCpu The cross context virtual CPU structure.
3585 *
3586 * @remarks No-long-jump zone!!!
3587 */
3588static int hmR0VmxExportGuestRflags(PVMCPU pVCpu)
3589{
3590 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3591 {
3592 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
3593
3594 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3595 Let us assert it as such and use 32-bit VMWRITE. */
3596 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
3597 X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
3598 Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3599 Assert(!(fEFlags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3600
3601 /*
3602 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3603 * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3604 * can run the real-mode guest code under Virtual 8086 mode.
3605 */
3606 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3607 {
3608 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3609 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3610 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3611 fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3612 fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3613 }
3614
3615 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3616 AssertRCReturn(rc, rc);
3617
3618 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3619 Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3620 }
3621 return VINF_SUCCESS;
3622}
3623
3624
3625/**
3626 * Exports the guest CR0 control register into the guest-state area in the VMCS.
3627 *
3628 * The guest FPU state is always pre-loaded hence we don't need to bother about
3629 * sharing FPU related CR0 bits between the guest and host.
3630 *
3631 * @returns VBox status code.
3632 * @param pVCpu The cross context virtual CPU structure.
3633 *
3634 * @remarks No-long-jump zone!!!
3635 */
3636static int hmR0VmxExportGuestCR0(PVMCPU pVCpu)
3637{
3638 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3639 {
3640 PVM pVM = pVCpu->CTX_SUFF(pVM);
3641 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
3642 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.cr0));
3643
3644 uint32_t const u32ShadowCr0 = pVCpu->cpum.GstCtx.cr0;
3645 uint32_t u32GuestCr0 = pVCpu->cpum.GstCtx.cr0;
3646
3647 /*
3648 * Setup VT-x's view of the guest CR0.
3649 * Minimize VM-exits due to CR3 changes when we have NestedPaging.
3650 */
3651 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3652 if (pVM->hm.s.fNestedPaging)
3653 {
3654 if (CPUMIsGuestPagingEnabled(pVCpu))
3655 {
3656 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3657 uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
3658 | VMX_PROC_CTLS_CR3_STORE_EXIT);
3659 }
3660 else
3661 {
3662 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3663 uProcCtls |= VMX_PROC_CTLS_CR3_LOAD_EXIT
3664 | VMX_PROC_CTLS_CR3_STORE_EXIT;
3665 }
3666
3667 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3668 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3669 uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
3670 }
3671 else
3672 {
3673 /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3674 u32GuestCr0 |= X86_CR0_WP;
3675 }
3676
3677 /*
3678 * Guest FPU bits.
3679 *
3680 * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3681 * using CR0.TS.
3682 *
3683 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3684 * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3685 */
3686 u32GuestCr0 |= X86_CR0_NE;
3687
3688 /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3689 bool const fInterceptMF = !(u32ShadowCr0 & X86_CR0_NE);
3690
3691 /*
3692 * Update exception intercepts.
3693 */
3694 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3695 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3696 {
3697 Assert(PDMVmmDevHeapIsEnabled(pVM));
3698 Assert(pVM->hm.s.vmx.pRealModeTSS);
3699 uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3700 }
3701 else
3702 {
3703 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3704 uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3705 if (fInterceptMF)
3706 uXcptBitmap |= RT_BIT(X86_XCPT_MF);
3707 }
3708
3709 /* Additional intercepts for debugging, define these yourself explicitly. */
3710#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3711 uXcptBitmap |= 0
3712 | RT_BIT(X86_XCPT_BP)
3713 | RT_BIT(X86_XCPT_DE)
3714 | RT_BIT(X86_XCPT_NM)
3715 | RT_BIT(X86_XCPT_TS)
3716 | RT_BIT(X86_XCPT_UD)
3717 | RT_BIT(X86_XCPT_NP)
3718 | RT_BIT(X86_XCPT_SS)
3719 | RT_BIT(X86_XCPT_GP)
3720 | RT_BIT(X86_XCPT_PF)
3721 | RT_BIT(X86_XCPT_MF)
3722 ;
3723#elif defined(HMVMX_ALWAYS_TRAP_PF)
3724 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
3725#endif
3726 Assert(pVM->hm.s.fNestedPaging || (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
3727
3728 /*
3729 * Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3730 */
3731 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3732 uint32_t fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3733 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3734 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
3735 else
3736 Assert((fSetCr0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3737
3738 u32GuestCr0 |= fSetCr0;
3739 u32GuestCr0 &= fZapCr0;
3740 u32GuestCr0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3741
3742 /*
3743 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3744 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3745 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3746 */
3747 uint32_t u32Cr0Mask = X86_CR0_PE
3748 | X86_CR0_NE
3749 | (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3750 | X86_CR0_PG
3751 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3752 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3753 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3754
3755 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3756 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3757 * and @bugref{6944}. */
3758#if 0
3759 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3760 u32Cr0Mask &= ~X86_CR0_PE;
3761#endif
3762 /*
3763 * Finally, update VMCS fields with the CR0 values and the exception bitmap.
3764 */
3765 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCr0);
3766 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32ShadowCr0);
3767 if (u32Cr0Mask != pVCpu->hm.s.vmx.u32Cr0Mask)
3768 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32Cr0Mask);
3769 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3770 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3771 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3772 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3773 AssertRCReturn(rc, rc);
3774
3775 /* Update our caches. */
3776 pVCpu->hm.s.vmx.u32Cr0Mask = u32Cr0Mask;
3777 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3778 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3779
3780 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3781
3782 Log4Func(("u32Cr0Mask=%#RX32 u32ShadowCr0=%#RX32 u32GuestCr0=%#RX32 (fSetCr0=%#RX32 fZapCr0=%#RX32\n", u32Cr0Mask,
3783 u32ShadowCr0, u32GuestCr0, fSetCr0, fZapCr0));
3784 }
3785
3786 return VINF_SUCCESS;
3787}
3788
3789
3790/**
3791 * Exports the guest control registers (CR3, CR4) into the guest-state area
3792 * in the VMCS.
3793 *
3794 * @returns VBox strict status code.
3795 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3796 * without unrestricted guest access and the VMMDev is not presently
3797 * mapped (e.g. EFI32).
3798 *
3799 * @param pVCpu The cross context virtual CPU structure.
3800 *
3801 * @remarks No-long-jump zone!!!
3802 */
3803static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu)
3804{
3805 int rc = VINF_SUCCESS;
3806 PVM pVM = pVCpu->CTX_SUFF(pVM);
3807
3808 /*
3809 * Guest CR2.
3810 * It's always loaded in the assembler code. Nothing to do here.
3811 */
3812
3813 /*
3814 * Guest CR3.
3815 */
3816 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3817 {
3818 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
3819
3820 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3821 if (pVM->hm.s.fNestedPaging)
3822 {
3823 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3824
3825 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3826 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3827 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3828 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3829
3830 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3831 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3832 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3833
3834 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3835 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3836 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3837 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3838 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3839 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3840 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3841
3842 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3843 AssertRCReturn(rc, rc);
3844
3845 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3846 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3847 || CPUMIsGuestPagingEnabledEx(pCtx))
3848 {
3849 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3850 if (CPUMIsGuestInPAEModeEx(pCtx))
3851 {
3852 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3853 AssertRCReturn(rc, rc);
3854 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3855 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3856 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3857 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3858 AssertRCReturn(rc, rc);
3859 }
3860
3861 /*
3862 * The guest's view of its CR3 is unblemished with Nested Paging when the
3863 * guest is using paging or we have unrestricted guest execution to handle
3864 * the guest when it's not using paging.
3865 */
3866 GCPhysGuestCR3 = pCtx->cr3;
3867 }
3868 else
3869 {
3870 /*
3871 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
3872 * thinks it accesses physical memory directly, we use our identity-mapped
3873 * page table to map guest-linear to guest-physical addresses. EPT takes care
3874 * of translating it to host-physical addresses.
3875 */
3876 RTGCPHYS GCPhys;
3877 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3878
3879 /* We obtain it here every time as the guest could have relocated this PCI region. */
3880 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3881 if (RT_SUCCESS(rc))
3882 { /* likely */ }
3883 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3884 {
3885 Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3886 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3887 }
3888 else
3889 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3890
3891 GCPhysGuestCR3 = GCPhys;
3892 }
3893
3894 Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
3895 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3896 AssertRCReturn(rc, rc);
3897 }
3898 else
3899 {
3900 /* Non-nested paging case, just use the hypervisor's CR3. */
3901 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3902
3903 Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
3904 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3905 AssertRCReturn(rc, rc);
3906 }
3907
3908 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3909 }
3910
3911 /*
3912 * Guest CR4.
3913 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3914 */
3915 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3916 {
3917 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3918 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
3919 Assert(!RT_HI_U32(pCtx->cr4));
3920
3921 uint32_t u32GuestCr4 = pCtx->cr4;
3922 uint32_t const u32ShadowCr4 = pCtx->cr4;
3923
3924 /*
3925 * Setup VT-x's view of the guest CR4.
3926 *
3927 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3928 * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3929 * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3930 *
3931 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3932 */
3933 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3934 {
3935 Assert(pVM->hm.s.vmx.pRealModeTSS);
3936 Assert(PDMVmmDevHeapIsEnabled(pVM));
3937 u32GuestCr4 &= ~X86_CR4_VME;
3938 }
3939
3940 if (pVM->hm.s.fNestedPaging)
3941 {
3942 if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3943 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3944 {
3945 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3946 u32GuestCr4 |= X86_CR4_PSE;
3947 /* Our identity mapping is a 32-bit page directory. */
3948 u32GuestCr4 &= ~X86_CR4_PAE;
3949 }
3950 /* else use guest CR4.*/
3951 }
3952 else
3953 {
3954 /*
3955 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3956 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3957 */
3958 switch (pVCpu->hm.s.enmShadowMode)
3959 {
3960 case PGMMODE_REAL: /* Real-mode. */
3961 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3962 case PGMMODE_32_BIT: /* 32-bit paging. */
3963 {
3964 u32GuestCr4 &= ~X86_CR4_PAE;
3965 break;
3966 }
3967
3968 case PGMMODE_PAE: /* PAE paging. */
3969 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3970 {
3971 u32GuestCr4 |= X86_CR4_PAE;
3972 break;
3973 }
3974
3975 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3976 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3977#ifdef VBOX_ENABLE_64_BITS_GUESTS
3978 break;
3979#endif
3980 default:
3981 AssertFailed();
3982 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3983 }
3984 }
3985
3986 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3987 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3988 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3989 u32GuestCr4 |= fSetCr4;
3990 u32GuestCr4 &= fZapCr4;
3991
3992 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them,
3993 that would cause a VM-exit. */
3994 uint32_t u32Cr4Mask = X86_CR4_VME
3995 | X86_CR4_PAE
3996 | X86_CR4_PGE
3997 | X86_CR4_PSE
3998 | X86_CR4_VMXE;
3999 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4000 u32Cr4Mask |= X86_CR4_OSXSAVE;
4001 if (pVM->cpum.ro.GuestFeatures.fPcid)
4002 u32Cr4Mask |= X86_CR4_PCIDE;
4003
4004 /* Write VT-x's view of the guest CR4, the CR4 modify mask and the read-only CR4 shadow
4005 into the VMCS and update our cache. */
4006 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCr4);
4007 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32ShadowCr4);
4008 if (pVCpu->hm.s.vmx.u32Cr4Mask != u32Cr4Mask)
4009 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32Cr4Mask);
4010 AssertRCReturn(rc, rc);
4011 pVCpu->hm.s.vmx.u32Cr4Mask = u32Cr4Mask;
4012
4013 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4014 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
4015
4016 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4017
4018 Log4Func(("u32GuestCr4=%#RX32 u32ShadowCr4=%#RX32 (fSetCr4=%#RX32 fZapCr4=%#RX32)\n", u32GuestCr4, u32ShadowCr4, fSetCr4,
4019 fZapCr4));
4020 }
4021 return rc;
4022}
4023
4024
4025/**
4026 * Exports the guest debug registers into the guest-state area in the VMCS.
4027 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4028 *
4029 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4030 *
4031 * @returns VBox status code.
4032 * @param pVCpu The cross context virtual CPU structure.
4033 *
4034 * @remarks No-long-jump zone!!!
4035 */
4036static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu)
4037{
4038 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4039
4040#ifdef VBOX_STRICT
4041 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4042 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
4043 {
4044 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4045 Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0);
4046 Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
4047 }
4048#endif
4049
4050 bool fSteppingDB = false;
4051 bool fInterceptMovDRx = false;
4052 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4053 if (pVCpu->hm.s.fSingleInstruction)
4054 {
4055 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4056 PVM pVM = pVCpu->CTX_SUFF(pVM);
4057 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
4058 {
4059 uProcCtls |= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
4060 Assert(fSteppingDB == false);
4061 }
4062 else
4063 {
4064 pVCpu->cpum.GstCtx.eflags.u32 |= X86_EFL_TF;
4065 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
4066 pVCpu->hm.s.fClearTrapFlag = true;
4067 fSteppingDB = true;
4068 }
4069 }
4070
4071 uint32_t u32GuestDr7;
4072 if ( fSteppingDB
4073 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4074 {
4075 /*
4076 * Use the combined guest and host DRx values found in the hypervisor register set
4077 * because the debugger has breakpoints active or someone is single stepping on the
4078 * host side without a monitor trap flag.
4079 *
4080 * Note! DBGF expects a clean DR6 state before executing guest code.
4081 */
4082#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4083 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4084 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4085 {
4086 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4087 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4088 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4089 }
4090 else
4091#endif
4092 if (!CPUMIsHyperDebugStateActive(pVCpu))
4093 {
4094 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4095 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4096 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4097 }
4098
4099 /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4100 u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4101 pVCpu->hm.s.fUsingHyperDR7 = true;
4102 fInterceptMovDRx = true;
4103 }
4104 else
4105 {
4106 /*
4107 * If the guest has enabled debug registers, we need to load them prior to
4108 * executing guest code so they'll trigger at the right time.
4109 */
4110 if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD))
4111 {
4112#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4113 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4114 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4115 {
4116 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4117 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4118 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4119 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4120 }
4121 else
4122#endif
4123 if (!CPUMIsGuestDebugStateActive(pVCpu))
4124 {
4125 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4126 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4127 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4128 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4129 }
4130 Assert(!fInterceptMovDRx);
4131 }
4132 /*
4133 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4134 * must intercept #DB in order to maintain a correct DR6 guest value, and
4135 * because we need to intercept it to prevent nested #DBs from hanging the
4136 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4137 */
4138#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4139 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4140 && !CPUMIsGuestDebugStateActive(pVCpu))
4141#else
4142 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4143#endif
4144 {
4145 fInterceptMovDRx = true;
4146 }
4147
4148 /* Update DR7 with the actual guest value. */
4149 u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
4150 pVCpu->hm.s.fUsingHyperDR7 = false;
4151 }
4152
4153 if (fInterceptMovDRx)
4154 uProcCtls |= VMX_PROC_CTLS_MOV_DR_EXIT;
4155 else
4156 uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
4157
4158 /*
4159 * Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
4160 * monitor-trap flag and update our cache.
4161 */
4162 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4163 {
4164 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4165 AssertRCReturn(rc2, rc2);
4166 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4167 }
4168
4169 /*
4170 * Update guest DR7.
4171 */
4172 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
4173 AssertRCReturn(rc, rc);
4174
4175 return VINF_SUCCESS;
4176}
4177
4178
4179#ifdef VBOX_STRICT
4180/**
4181 * Strict function to validate segment registers.
4182 *
4183 * @param pVCpu The cross context virtual CPU structure.
4184 *
4185 * @remarks Will import guest CR0 on strict builds during validation of
4186 * segments.
4187 */
4188static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu)
4189{
4190 /*
4191 * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4192 *
4193 * The reason we check for attribute value 0 in this function and not just the unusable bit is
4194 * because hmR0VmxExportGuestSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4195 * and doesn't change the guest-context value.
4196 */
4197 PVM pVM = pVCpu->CTX_SUFF(pVM);
4198 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4199 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4200 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4201 && ( !CPUMIsGuestInRealModeEx(pCtx)
4202 && !CPUMIsGuestInV86ModeEx(pCtx)))
4203 {
4204 /* Protected mode checks */
4205 /* CS */
4206 Assert(pCtx->cs.Attr.n.u1Present);
4207 Assert(!(pCtx->cs.Attr.u & 0xf00));
4208 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4209 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4210 || !(pCtx->cs.Attr.n.u1Granularity));
4211 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4212 || (pCtx->cs.Attr.n.u1Granularity));
4213 /* CS cannot be loaded with NULL in protected mode. */
4214 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4215 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4216 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4217 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4218 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4219 else
4220 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4221 /* SS */
4222 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4223 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4224 if ( !(pCtx->cr0 & X86_CR0_PE)
4225 || pCtx->cs.Attr.n.u4Type == 3)
4226 {
4227 Assert(!pCtx->ss.Attr.n.u2Dpl);
4228 }
4229 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4230 {
4231 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4232 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4233 Assert(pCtx->ss.Attr.n.u1Present);
4234 Assert(!(pCtx->ss.Attr.u & 0xf00));
4235 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4236 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4237 || !(pCtx->ss.Attr.n.u1Granularity));
4238 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4239 || (pCtx->ss.Attr.n.u1Granularity));
4240 }
4241 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmentReg(). */
4242 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4243 {
4244 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4245 Assert(pCtx->ds.Attr.n.u1Present);
4246 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4247 Assert(!(pCtx->ds.Attr.u & 0xf00));
4248 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4249 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4250 || !(pCtx->ds.Attr.n.u1Granularity));
4251 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4252 || (pCtx->ds.Attr.n.u1Granularity));
4253 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4254 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4255 }
4256 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4257 {
4258 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4259 Assert(pCtx->es.Attr.n.u1Present);
4260 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4261 Assert(!(pCtx->es.Attr.u & 0xf00));
4262 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4263 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4264 || !(pCtx->es.Attr.n.u1Granularity));
4265 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4266 || (pCtx->es.Attr.n.u1Granularity));
4267 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4268 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4269 }
4270 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4271 {
4272 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4273 Assert(pCtx->fs.Attr.n.u1Present);
4274 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4275 Assert(!(pCtx->fs.Attr.u & 0xf00));
4276 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4277 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4278 || !(pCtx->fs.Attr.n.u1Granularity));
4279 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4280 || (pCtx->fs.Attr.n.u1Granularity));
4281 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4282 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4283 }
4284 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4285 {
4286 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4287 Assert(pCtx->gs.Attr.n.u1Present);
4288 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4289 Assert(!(pCtx->gs.Attr.u & 0xf00));
4290 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4291 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4292 || !(pCtx->gs.Attr.n.u1Granularity));
4293 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4294 || (pCtx->gs.Attr.n.u1Granularity));
4295 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4296 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4297 }
4298 /* 64-bit capable CPUs. */
4299# if HC_ARCH_BITS == 64
4300 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4301 Assert(!pCtx->ss.Attr.u || !RT_HI_U32(pCtx->ss.u64Base));
4302 Assert(!pCtx->ds.Attr.u || !RT_HI_U32(pCtx->ds.u64Base));
4303 Assert(!pCtx->es.Attr.u || !RT_HI_U32(pCtx->es.u64Base));
4304# endif
4305 }
4306 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4307 || ( CPUMIsGuestInRealModeEx(pCtx)
4308 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4309 {
4310 /* Real and v86 mode checks. */
4311 /* hmR0VmxExportGuestSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4312 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4313 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4314 {
4315 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4316 }
4317 else
4318 {
4319 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4320 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4321 }
4322
4323 /* CS */
4324 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4325 Assert(pCtx->cs.u32Limit == 0xffff);
4326 Assert(u32CSAttr == 0xf3);
4327 /* SS */
4328 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4329 Assert(pCtx->ss.u32Limit == 0xffff);
4330 Assert(u32SSAttr == 0xf3);
4331 /* DS */
4332 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4333 Assert(pCtx->ds.u32Limit == 0xffff);
4334 Assert(u32DSAttr == 0xf3);
4335 /* ES */
4336 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4337 Assert(pCtx->es.u32Limit == 0xffff);
4338 Assert(u32ESAttr == 0xf3);
4339 /* FS */
4340 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4341 Assert(pCtx->fs.u32Limit == 0xffff);
4342 Assert(u32FSAttr == 0xf3);
4343 /* GS */
4344 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4345 Assert(pCtx->gs.u32Limit == 0xffff);
4346 Assert(u32GSAttr == 0xf3);
4347 /* 64-bit capable CPUs. */
4348# if HC_ARCH_BITS == 64
4349 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4350 Assert(!u32SSAttr || !RT_HI_U32(pCtx->ss.u64Base));
4351 Assert(!u32DSAttr || !RT_HI_U32(pCtx->ds.u64Base));
4352 Assert(!u32ESAttr || !RT_HI_U32(pCtx->es.u64Base));
4353# endif
4354 }
4355}
4356#endif /* VBOX_STRICT */
4357
4358
4359/**
4360 * Exports a guest segment register into the guest-state area in the VMCS.
4361 *
4362 * @returns VBox status code.
4363 * @param pVCpu The cross context virtual CPU structure.
4364 * @param idxSel Index of the selector in the VMCS.
4365 * @param idxLimit Index of the segment limit in the VMCS.
4366 * @param idxBase Index of the segment base in the VMCS.
4367 * @param idxAccess Index of the access rights of the segment in the VMCS.
4368 * @param pSelReg Pointer to the segment selector.
4369 *
4370 * @remarks No-long-jump zone!!!
4371 */
4372static int hmR0VmxExportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
4373 PCCPUMSELREG pSelReg)
4374{
4375 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4376 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4377 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4378 AssertRCReturn(rc, rc);
4379
4380 uint32_t u32Access = pSelReg->Attr.u;
4381 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4382 {
4383 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4384 u32Access = 0xf3;
4385 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4386 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4387 }
4388 else
4389 {
4390 /*
4391 * The way to differentiate between whether this is really a null selector or was just
4392 * a selector loaded with 0 in real-mode is using the segment attributes. A selector
4393 * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4394 * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4395 * NULL selectors loaded in protected-mode have their attribute as 0.
4396 */
4397 if (!u32Access)
4398 u32Access = X86DESCATTR_UNUSABLE;
4399 }
4400
4401 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4402 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4403 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4404
4405 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4406 AssertRCReturn(rc, rc);
4407 return rc;
4408}
4409
4410
4411/**
4412 * Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4413 * into the guest-state area in the VMCS.
4414 *
4415 * @returns VBox status code.
4416 * @param pVCpu The cross context virtual CPU structure.
4417 *
4418 * @remarks Will import guest CR0 on strict builds during validation of
4419 * segments.
4420 * @remarks No-long-jump zone!!!
4421 */
4422static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu)
4423{
4424 int rc = VERR_INTERNAL_ERROR_5;
4425 PVM pVM = pVCpu->CTX_SUFF(pVM);
4426 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4427
4428 /*
4429 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4430 */
4431 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4432 {
4433#ifdef VBOX_WITH_REM
4434 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4435 {
4436 Assert(pVM->hm.s.vmx.pRealModeTSS);
4437 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4438 if ( pVCpu->hm.s.vmx.fWasInRealMode
4439 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4440 {
4441 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4442 in real-mode (e.g. OpenBSD 4.0) */
4443 REMFlushTBs(pVM);
4444 Log4Func(("Switch to protected mode detected!\n"));
4445 pVCpu->hm.s.vmx.fWasInRealMode = false;
4446 }
4447 }
4448#endif
4449 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
4450 {
4451 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
4452 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4453 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pCtx->cs.Attr.u;
4454 rc = HMVMX_EXPORT_SREG(CS, &pCtx->cs);
4455 AssertRCReturn(rc, rc);
4456 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
4457 }
4458
4459 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
4460 {
4461 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
4462 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4463 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pCtx->ss.Attr.u;
4464 rc = HMVMX_EXPORT_SREG(SS, &pCtx->ss);
4465 AssertRCReturn(rc, rc);
4466 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
4467 }
4468
4469 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
4470 {
4471 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
4472 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4473 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pCtx->ds.Attr.u;
4474 rc = HMVMX_EXPORT_SREG(DS, &pCtx->ds);
4475 AssertRCReturn(rc, rc);
4476 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
4477 }
4478
4479 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
4480 {
4481 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
4482 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4483 pVCpu->hm.s.vmx.RealMode.AttrES.u = pCtx->es.Attr.u;
4484 rc = HMVMX_EXPORT_SREG(ES, &pCtx->es);
4485 AssertRCReturn(rc, rc);
4486 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
4487 }
4488
4489 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
4490 {
4491 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
4492 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4493 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pCtx->fs.Attr.u;
4494 rc = HMVMX_EXPORT_SREG(FS, &pCtx->fs);
4495 AssertRCReturn(rc, rc);
4496 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
4497 }
4498
4499 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
4500 {
4501 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
4502 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4503 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pCtx->gs.Attr.u;
4504 rc = HMVMX_EXPORT_SREG(GS, &pCtx->gs);
4505 AssertRCReturn(rc, rc);
4506 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
4507 }
4508
4509#ifdef VBOX_STRICT
4510 hmR0VmxValidateSegmentRegs(pVCpu);
4511#endif
4512
4513 Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pCtx->cs.Sel, pCtx->cs.u64Base,
4514 pCtx->cs.u32Limit, pCtx->cs.Attr.u));
4515 }
4516
4517 /*
4518 * Guest TR.
4519 */
4520 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4521 {
4522 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
4523
4524 /*
4525 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
4526 * achieved using the interrupt redirection bitmap (all bits cleared to let the guest
4527 * handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4528 */
4529 uint16_t u16Sel = 0;
4530 uint32_t u32Limit = 0;
4531 uint64_t u64Base = 0;
4532 uint32_t u32AccessRights = 0;
4533
4534 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4535 {
4536 u16Sel = pCtx->tr.Sel;
4537 u32Limit = pCtx->tr.u32Limit;
4538 u64Base = pCtx->tr.u64Base;
4539 u32AccessRights = pCtx->tr.Attr.u;
4540 }
4541 else
4542 {
4543 Assert(pVM->hm.s.vmx.pRealModeTSS);
4544 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
4545
4546 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4547 RTGCPHYS GCPhys;
4548 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4549 AssertRCReturn(rc, rc);
4550
4551 X86DESCATTR DescAttr;
4552 DescAttr.u = 0;
4553 DescAttr.n.u1Present = 1;
4554 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4555
4556 u16Sel = 0;
4557 u32Limit = HM_VTX_TSS_SIZE;
4558 u64Base = GCPhys; /* in real-mode phys = virt. */
4559 u32AccessRights = DescAttr.u;
4560 }
4561
4562 /* Validate. */
4563 Assert(!(u16Sel & RT_BIT(2)));
4564 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4565 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4566 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4567 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4568 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4569 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4570 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4571 Assert( (u32Limit & 0xfff) == 0xfff
4572 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4573 Assert( !(pCtx->tr.u32Limit & 0xfff00000)
4574 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4575
4576 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4577 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4578 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4579 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4580 AssertRCReturn(rc, rc);
4581
4582 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4583 Log4Func(("TR base=%#RX64\n", pCtx->tr.u64Base));
4584 }
4585
4586 /*
4587 * Guest GDTR.
4588 */
4589 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4590 {
4591 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
4592
4593 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
4594 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
4595 AssertRCReturn(rc, rc);
4596
4597 /* Validate. */
4598 Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4599
4600 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4601 Log4Func(("GDTR base=%#RX64\n", pCtx->gdtr.pGdt));
4602 }
4603
4604 /*
4605 * Guest LDTR.
4606 */
4607 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4608 {
4609 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
4610
4611 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4612 uint32_t u32Access = 0;
4613 if (!pCtx->ldtr.Attr.u)
4614 u32Access = X86DESCATTR_UNUSABLE;
4615 else
4616 u32Access = pCtx->ldtr.Attr.u;
4617
4618 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
4619 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
4620 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4621 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
4622 AssertRCReturn(rc, rc);
4623
4624 /* Validate. */
4625 if (!(u32Access & X86DESCATTR_UNUSABLE))
4626 {
4627 Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4628 Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4629 Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4630 Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4631 Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4632 Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4633 Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
4634 || !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4635 Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
4636 || pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4637 }
4638
4639 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4640 Log4Func(("LDTR base=%#RX64\n", pCtx->ldtr.u64Base));
4641 }
4642
4643 /*
4644 * Guest IDTR.
4645 */
4646 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4647 {
4648 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
4649
4650 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
4651 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
4652 AssertRCReturn(rc, rc);
4653
4654 /* Validate. */
4655 Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4656
4657 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4658 Log4Func(("IDTR base=%#RX64\n", pCtx->idtr.pIdt));
4659 }
4660
4661 return VINF_SUCCESS;
4662}
4663
4664
4665/**
4666 * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4667 * areas.
4668 *
4669 * These MSRs will automatically be loaded to the host CPU on every successful
4670 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4671 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4672 * -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4673 *
4674 * Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4675 *
4676 * @returns VBox status code.
4677 * @param pVCpu The cross context virtual CPU structure.
4678 *
4679 * @remarks No-long-jump zone!!!
4680 */
4681static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu)
4682{
4683 AssertPtr(pVCpu);
4684 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4685
4686 /*
4687 * MSRs that we use the auto-load/store MSR area in the VMCS.
4688 * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4689 */
4690 PVM pVM = pVCpu->CTX_SUFF(pVM);
4691 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4692 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4693 {
4694 if (pVM->hm.s.fAllow64BitGuests)
4695 {
4696#if HC_ARCH_BITS == 32
4697 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS | CPUMCTX_EXTRN_KERNEL_GS_BASE);
4698
4699 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pCtx->msrLSTAR, false, NULL);
4700 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pCtx->msrSTAR, false, NULL);
4701 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pCtx->msrSFMASK, false, NULL);
4702 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, false, NULL);
4703 AssertRCReturn(rc, rc);
4704# ifdef LOG_ENABLED
4705 PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4706 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4707 Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4708# endif
4709#endif
4710 }
4711 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4712 }
4713
4714 /*
4715 * Guest Sysenter MSRs.
4716 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4717 * VM-exits on WRMSRs for these MSRs.
4718 */
4719 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4720 {
4721 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
4722
4723 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4724 {
4725 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
4726 AssertRCReturn(rc, rc);
4727 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4728 }
4729
4730 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4731 {
4732 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
4733 AssertRCReturn(rc, rc);
4734 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4735 }
4736
4737 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4738 {
4739 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
4740 AssertRCReturn(rc, rc);
4741 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4742 }
4743 }
4744
4745 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4746 {
4747 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
4748
4749 if (hmR0VmxShouldSwapEferMsr(pVCpu))
4750 {
4751 /*
4752 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4753 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4754 */
4755 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4756 {
4757 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
4758 AssertRCReturn(rc,rc);
4759 Log4Func(("EFER=%#RX64\n", pCtx->msrEFER));
4760 }
4761 else
4762 {
4763 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pCtx->msrEFER, false /* fUpdateHostMsr */,
4764 NULL /* pfAddedAndUpdated */);
4765 AssertRCReturn(rc, rc);
4766
4767 /* We need to intercept reads too, see @bugref{7386#c16}. */
4768 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
4769 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4770 Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pCtx->msrEFER,
4771 pVCpu->hm.s.vmx.cMsrs));
4772 }
4773 }
4774 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4775 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4776 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4777 }
4778
4779 return VINF_SUCCESS;
4780}
4781
4782
4783#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4784/**
4785 * Check if guest state allows safe use of 32-bit switcher again.
4786 *
4787 * Segment bases and protected mode structures must be 32-bit addressable
4788 * because the 32-bit switcher will ignore high dword when writing these VMCS
4789 * fields. See @bugref{8432} for details.
4790 *
4791 * @returns true if safe, false if must continue to use the 64-bit switcher.
4792 * @param pCtx Pointer to the guest-CPU context.
4793 *
4794 * @remarks No-long-jump zone!!!
4795 */
4796static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
4797{
4798 if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4799 if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4800 if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4801 if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4802 if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4803 if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4804 if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4805 if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4806 if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4807 if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4808
4809 /* All good, bases are 32-bit. */
4810 return true;
4811}
4812#endif
4813
4814
4815/**
4816 * Selects up the appropriate function to run guest code.
4817 *
4818 * @returns VBox status code.
4819 * @param pVCpu The cross context virtual CPU structure.
4820 *
4821 * @remarks No-long-jump zone!!!
4822 */
4823static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu)
4824{
4825 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4826 if (CPUMIsGuestInLongModeEx(pCtx))
4827 {
4828#ifndef VBOX_ENABLE_64_BITS_GUESTS
4829 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4830#endif
4831 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4832#if HC_ARCH_BITS == 32
4833 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4834 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4835 {
4836#ifdef VBOX_STRICT
4837 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4838 {
4839 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4840 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4841 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4842 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4843 | HM_CHANGED_VMX_ENTRY_CTLS
4844 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4845 }
4846#endif
4847 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4848
4849 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4850 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4851 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4852 Log4Func(("Selected 64-bit switcher\n"));
4853 }
4854#else
4855 /* 64-bit host. */
4856 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4857#endif
4858 }
4859 else
4860 {
4861 /* Guest is not in long mode, use the 32-bit handler. */
4862#if HC_ARCH_BITS == 32
4863 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4864 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4865 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4866 {
4867# ifdef VBOX_STRICT
4868 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4869 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4870 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4871 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4872 | HM_CHANGED_VMX_ENTRY_CTLS
4873 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4874# endif
4875 }
4876# ifdef VBOX_ENABLE_64_BITS_GUESTS
4877 /*
4878 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4879 * design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4880 * switcher flag because now we know the guest is in a sane state where it's safe
4881 * to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4882 * the much faster 32-bit switcher again.
4883 */
4884 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4885 {
4886 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4887 Log4Func(("Selected 32-bit switcher\n"));
4888 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4889 }
4890 else
4891 {
4892 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4893 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4894 || hmR0VmxIs32BitSwitcherSafe(pCtx))
4895 {
4896 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4897 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4898 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4899 | HM_CHANGED_VMX_ENTRY_CTLS
4900 | HM_CHANGED_VMX_EXIT_CTLS
4901 | HM_CHANGED_HOST_CONTEXT);
4902 Log4Func(("Selected 32-bit switcher (safe)\n"));
4903 }
4904 }
4905# else
4906 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4907# endif
4908#else
4909 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4910#endif
4911 }
4912 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4913 return VINF_SUCCESS;
4914}
4915
4916
4917/**
4918 * Wrapper for running the guest code in VT-x.
4919 *
4920 * @returns VBox status code, no informational status codes.
4921 * @param pVCpu The cross context virtual CPU structure.
4922 *
4923 * @remarks No-long-jump zone!!!
4924 */
4925DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu)
4926{
4927 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4928 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4929 pCtx->fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
4930
4931 /*
4932 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4933 * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4934 * callee-saved and thus the need for this XMM wrapper.
4935 *
4936 * See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4937 */
4938 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4939 /** @todo Add stats for resume vs launch. */
4940 PVM pVM = pVCpu->CTX_SUFF(pVM);
4941#ifdef VBOX_WITH_KERNEL_USING_XMM
4942 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4943#else
4944 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4945#endif
4946 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4947 return rc;
4948}
4949
4950
4951/**
4952 * Reports world-switch error and dumps some useful debug info.
4953 *
4954 * @param pVCpu The cross context virtual CPU structure.
4955 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4956 * @param pVmxTransient Pointer to the VMX transient structure (only
4957 * exitReason updated).
4958 */
4959static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
4960{
4961 Assert(pVCpu);
4962 Assert(pVmxTransient);
4963 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
4964
4965 Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
4966 switch (rcVMRun)
4967 {
4968 case VERR_VMX_INVALID_VMXON_PTR:
4969 AssertFailed();
4970 break;
4971 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4972 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4973 {
4974 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4975 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4976 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
4977 AssertRC(rc);
4978
4979 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4980 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4981 Cannot do it here as we may have been long preempted. */
4982
4983#ifdef VBOX_STRICT
4984 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4985 pVmxTransient->uExitReason));
4986 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
4987 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4988 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4989 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4990 else
4991 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4992 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4993 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4994
4995 /* VMX control bits. */
4996 uint32_t u32Val;
4997 uint64_t u64Val;
4998 RTHCUINTREG uHCReg;
4999 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5000 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5001 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5002 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5003 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
5004 {
5005 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5006 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5007 }
5008 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5009 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5010 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5011 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5012 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5013 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5014 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5015 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5016 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5017 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5018 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5019 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5020 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5021 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5022 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5023 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5024 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5025 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5026 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5027 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5028 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5029 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5030 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5031 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5032 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5033 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5034 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5035 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5036 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5037 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5038 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5039 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5040 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5041 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5042 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5043 {
5044 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5045 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5046 }
5047
5048 /* Guest bits. */
5049 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5050 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
5051 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5052 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
5053 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5054 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
5055 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
5056 {
5057 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5058 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5059 }
5060
5061 /* Host bits. */
5062 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5063 Log4(("Host CR0 %#RHr\n", uHCReg));
5064 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5065 Log4(("Host CR3 %#RHr\n", uHCReg));
5066 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5067 Log4(("Host CR4 %#RHr\n", uHCReg));
5068
5069 RTGDTR HostGdtr;
5070 PCX86DESCHC pDesc;
5071 ASMGetGDTR(&HostGdtr);
5072 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5073 Log4(("Host CS %#08x\n", u32Val));
5074 if (u32Val < HostGdtr.cbGdt)
5075 {
5076 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5077 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5078 }
5079
5080 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5081 Log4(("Host DS %#08x\n", u32Val));
5082 if (u32Val < HostGdtr.cbGdt)
5083 {
5084 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5085 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5086 }
5087
5088 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5089 Log4(("Host ES %#08x\n", u32Val));
5090 if (u32Val < HostGdtr.cbGdt)
5091 {
5092 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5093 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5094 }
5095
5096 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5097 Log4(("Host FS %#08x\n", u32Val));
5098 if (u32Val < HostGdtr.cbGdt)
5099 {
5100 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5101 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5102 }
5103
5104 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5105 Log4(("Host GS %#08x\n", u32Val));
5106 if (u32Val < HostGdtr.cbGdt)
5107 {
5108 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5109 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5110 }
5111
5112 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5113 Log4(("Host SS %#08x\n", u32Val));
5114 if (u32Val < HostGdtr.cbGdt)
5115 {
5116 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5117 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5118 }
5119
5120 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5121 Log4(("Host TR %#08x\n", u32Val));
5122 if (u32Val < HostGdtr.cbGdt)
5123 {
5124 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5125 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5126 }
5127
5128 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5129 Log4(("Host TR Base %#RHv\n", uHCReg));
5130 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5131 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5132 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5133 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5134 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5135 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5136 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5137 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5138 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5139 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5140 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5141 Log4(("Host RSP %#RHv\n", uHCReg));
5142 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5143 Log4(("Host RIP %#RHv\n", uHCReg));
5144# if HC_ARCH_BITS == 64
5145 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5146 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5147 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5148 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5149 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5150 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5151# endif
5152#endif /* VBOX_STRICT */
5153 break;
5154 }
5155
5156 default:
5157 /* Impossible */
5158 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5159 break;
5160 }
5161}
5162
5163
5164#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5165#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5166# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5167#endif
5168#ifdef VBOX_STRICT
5169static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5170{
5171 switch (idxField)
5172 {
5173 case VMX_VMCS_GUEST_RIP:
5174 case VMX_VMCS_GUEST_RSP:
5175 case VMX_VMCS_GUEST_SYSENTER_EIP:
5176 case VMX_VMCS_GUEST_SYSENTER_ESP:
5177 case VMX_VMCS_GUEST_GDTR_BASE:
5178 case VMX_VMCS_GUEST_IDTR_BASE:
5179 case VMX_VMCS_GUEST_CS_BASE:
5180 case VMX_VMCS_GUEST_DS_BASE:
5181 case VMX_VMCS_GUEST_ES_BASE:
5182 case VMX_VMCS_GUEST_FS_BASE:
5183 case VMX_VMCS_GUEST_GS_BASE:
5184 case VMX_VMCS_GUEST_SS_BASE:
5185 case VMX_VMCS_GUEST_LDTR_BASE:
5186 case VMX_VMCS_GUEST_TR_BASE:
5187 case VMX_VMCS_GUEST_CR3:
5188 return true;
5189 }
5190 return false;
5191}
5192
5193static bool hmR0VmxIsValidReadField(uint32_t idxField)
5194{
5195 switch (idxField)
5196 {
5197 /* Read-only fields. */
5198 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5199 return true;
5200 }
5201 /* Remaining readable fields should also be writable. */
5202 return hmR0VmxIsValidWriteField(idxField);
5203}
5204#endif /* VBOX_STRICT */
5205
5206
5207/**
5208 * Executes the specified handler in 64-bit mode.
5209 *
5210 * @returns VBox status code (no informational status codes).
5211 * @param pVCpu The cross context virtual CPU structure.
5212 * @param enmOp The operation to perform.
5213 * @param cParams Number of parameters.
5214 * @param paParam Array of 32-bit parameters.
5215 */
5216VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
5217{
5218 PVM pVM = pVCpu->CTX_SUFF(pVM);
5219 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5220 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5221 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5222 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5223
5224#ifdef VBOX_STRICT
5225 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5226 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5227
5228 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5229 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5230#endif
5231
5232 /* Disable interrupts. */
5233 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5234
5235#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5236 RTCPUID idHostCpu = RTMpCpuId();
5237 CPUMR0SetLApic(pVCpu, idHostCpu);
5238#endif
5239
5240 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5241 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5242
5243 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5244 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5245 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5246
5247 /* Leave VMX Root Mode. */
5248 VMXDisable();
5249
5250 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5251
5252 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5253 CPUMSetHyperEIP(pVCpu, enmOp);
5254 for (int i = (int)cParams - 1; i >= 0; i--)
5255 CPUMPushHyper(pVCpu, paParam[i]);
5256
5257 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5258
5259 /* Call the switcher. */
5260 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
5261 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5262
5263 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5264 /* Make sure the VMX instructions don't cause #UD faults. */
5265 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5266
5267 /* Re-enter VMX Root Mode */
5268 int rc2 = VMXEnable(HCPhysCpuPage);
5269 if (RT_FAILURE(rc2))
5270 {
5271 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5272 ASMSetFlags(fOldEFlags);
5273 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5274 return rc2;
5275 }
5276
5277 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5278 AssertRC(rc2);
5279 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5280 Assert(!(ASMGetFlags() & X86_EFL_IF));
5281 ASMSetFlags(fOldEFlags);
5282 return rc;
5283}
5284
5285
5286/**
5287 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5288 * supporting 64-bit guests.
5289 *
5290 * @returns VBox status code.
5291 * @param fResume Whether to VMLAUNCH or VMRESUME.
5292 * @param pCtx Pointer to the guest-CPU context.
5293 * @param pCache Pointer to the VMCS cache.
5294 * @param pVM The cross context VM structure.
5295 * @param pVCpu The cross context virtual CPU structure.
5296 */
5297DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5298{
5299 NOREF(fResume);
5300
5301 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5302 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5303
5304#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5305 pCache->uPos = 1;
5306 pCache->interPD = PGMGetInterPaeCR3(pVM);
5307 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5308#endif
5309
5310#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5311 pCache->TestIn.HCPhysCpuPage = 0;
5312 pCache->TestIn.HCPhysVmcs = 0;
5313 pCache->TestIn.pCache = 0;
5314 pCache->TestOut.HCPhysVmcs = 0;
5315 pCache->TestOut.pCache = 0;
5316 pCache->TestOut.pCtx = 0;
5317 pCache->TestOut.eflags = 0;
5318#else
5319 NOREF(pCache);
5320#endif
5321
5322 uint32_t aParam[10];
5323 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5324 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5325 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5326 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5327 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5328 aParam[5] = 0;
5329 aParam[6] = VM_RC_ADDR(pVM, pVM);
5330 aParam[7] = 0;
5331 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5332 aParam[9] = 0;
5333
5334#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5335 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5336 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5337#endif
5338 int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5339
5340#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5341 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5342 Assert(pCtx->dr[4] == 10);
5343 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5344#endif
5345
5346#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5347 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5348 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5349 pVCpu->hm.s.vmx.HCPhysVmcs));
5350 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5351 pCache->TestOut.HCPhysVmcs));
5352 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5353 pCache->TestOut.pCache));
5354 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5355 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5356 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5357 pCache->TestOut.pCtx));
5358 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5359#endif
5360 NOREF(pCtx);
5361 return rc;
5362}
5363
5364
5365/**
5366 * Initialize the VMCS-Read cache.
5367 *
5368 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5369 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5370 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5371 * (those that have a 32-bit FULL & HIGH part).
5372 *
5373 * @returns VBox status code.
5374 * @param pVCpu The cross context virtual CPU structure.
5375 */
5376static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
5377{
5378#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5379 do { \
5380 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5381 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5382 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5383 ++cReadFields; \
5384 } while (0)
5385
5386 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5387 uint32_t cReadFields = 0;
5388
5389 /*
5390 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5391 * and serve to indicate exceptions to the rules.
5392 */
5393
5394 /* Guest-natural selector base fields. */
5395#if 0
5396 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5397 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5398 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5399#endif
5400 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5401 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5402 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5403 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5404 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5405 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5406 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5407 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5408 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5409 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5410 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5411 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5412#if 0
5413 /* Unused natural width guest-state fields. */
5414 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5415 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5416#endif
5417 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5418 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5419
5420 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5421 these 64-bit fields (using "FULL" and "HIGH" fields). */
5422#if 0
5423 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5424 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5425 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5426 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5427 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5428 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5429 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5430 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5431 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5432#endif
5433
5434 /* Natural width guest-state fields. */
5435 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5436#if 0
5437 /* Currently unused field. */
5438 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5439#endif
5440
5441 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5442 {
5443 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5444 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5445 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5446 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5447 }
5448 else
5449 {
5450 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5451 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5452 }
5453
5454#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5455 return VINF_SUCCESS;
5456}
5457
5458
5459/**
5460 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5461 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5462 * darwin, running 64-bit guests).
5463 *
5464 * @returns VBox status code.
5465 * @param pVCpu The cross context virtual CPU structure.
5466 * @param idxField The VMCS field encoding.
5467 * @param u64Val 16, 32 or 64-bit value.
5468 */
5469VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5470{
5471 int rc;
5472 switch (idxField)
5473 {
5474 /*
5475 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5476 */
5477 /* 64-bit Control fields. */
5478 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5479 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5480 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5481 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5482 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5483 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5484 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5485 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5486 case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
5487 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5488 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5489 case VMX_VMCS64_CTRL_EPTP_FULL:
5490 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5491 /* 64-bit Guest-state fields. */
5492 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5493 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5494 case VMX_VMCS64_GUEST_PAT_FULL:
5495 case VMX_VMCS64_GUEST_EFER_FULL:
5496 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5497 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5498 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5499 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5500 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5501 /* 64-bit Host-state fields. */
5502 case VMX_VMCS64_HOST_PAT_FULL:
5503 case VMX_VMCS64_HOST_EFER_FULL:
5504 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5505 {
5506 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5507 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5508 break;
5509 }
5510
5511 /*
5512 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5513 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5514 */
5515 /* Natural-width Guest-state fields. */
5516 case VMX_VMCS_GUEST_CR3:
5517 case VMX_VMCS_GUEST_ES_BASE:
5518 case VMX_VMCS_GUEST_CS_BASE:
5519 case VMX_VMCS_GUEST_SS_BASE:
5520 case VMX_VMCS_GUEST_DS_BASE:
5521 case VMX_VMCS_GUEST_FS_BASE:
5522 case VMX_VMCS_GUEST_GS_BASE:
5523 case VMX_VMCS_GUEST_LDTR_BASE:
5524 case VMX_VMCS_GUEST_TR_BASE:
5525 case VMX_VMCS_GUEST_GDTR_BASE:
5526 case VMX_VMCS_GUEST_IDTR_BASE:
5527 case VMX_VMCS_GUEST_RSP:
5528 case VMX_VMCS_GUEST_RIP:
5529 case VMX_VMCS_GUEST_SYSENTER_ESP:
5530 case VMX_VMCS_GUEST_SYSENTER_EIP:
5531 {
5532 if (!(RT_HI_U32(u64Val)))
5533 {
5534 /* If this field is 64-bit, VT-x will zero out the top bits. */
5535 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5536 }
5537 else
5538 {
5539 /* Assert that only the 32->64 switcher case should ever come here. */
5540 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5541 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5542 }
5543 break;
5544 }
5545
5546 default:
5547 {
5548 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5549 rc = VERR_INVALID_PARAMETER;
5550 break;
5551 }
5552 }
5553 AssertRCReturn(rc, rc);
5554 return rc;
5555}
5556
5557
5558/**
5559 * Queue up a VMWRITE by using the VMCS write cache.
5560 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5561 *
5562 * @param pVCpu The cross context virtual CPU structure.
5563 * @param idxField The VMCS field encoding.
5564 * @param u64Val 16, 32 or 64-bit value.
5565 */
5566VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5567{
5568 AssertPtr(pVCpu);
5569 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5570
5571 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5572 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5573
5574 /* Make sure there are no duplicates. */
5575 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5576 {
5577 if (pCache->Write.aField[i] == idxField)
5578 {
5579 pCache->Write.aFieldVal[i] = u64Val;
5580 return VINF_SUCCESS;
5581 }
5582 }
5583
5584 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5585 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5586 pCache->Write.cValidEntries++;
5587 return VINF_SUCCESS;
5588}
5589#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5590
5591
5592/**
5593 * Sets up the usage of TSC-offsetting and updates the VMCS.
5594 *
5595 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5596 * VMX preemption timer.
5597 *
5598 * @returns VBox status code.
5599 * @param pVCpu The cross context virtual CPU structure.
5600 *
5601 * @remarks No-long-jump zone!!!
5602 */
5603static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5604{
5605 bool fOffsettedTsc;
5606 bool fParavirtTsc;
5607 PVM pVM = pVCpu->CTX_SUFF(pVM);
5608 uint64_t uTscOffset;
5609 if (pVM->hm.s.vmx.fUsePreemptTimer)
5610 {
5611 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
5612
5613 /* Make sure the returned values have sane upper and lower boundaries. */
5614 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5615 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5616 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5617 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5618
5619 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5620 int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
5621 AssertRC(rc);
5622 }
5623 else
5624 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
5625
5626 if (fParavirtTsc)
5627 {
5628 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5629 information before every VM-entry, hence disable it for performance sake. */
5630#if 0
5631 int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5632 AssertRC(rc);
5633#endif
5634 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5635 }
5636
5637 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
5638 if ( fOffsettedTsc
5639 && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5640 {
5641 if (pVCpu->hm.s.vmx.u64TscOffset != uTscOffset)
5642 {
5643 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
5644 AssertRC(rc);
5645 pVCpu->hm.s.vmx.u64TscOffset = uTscOffset;
5646 }
5647
5648 if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
5649 {
5650 uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
5651 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5652 AssertRC(rc);
5653 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5654 }
5655 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5656 }
5657 else
5658 {
5659 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5660 if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
5661 {
5662 uProcCtls |= VMX_PROC_CTLS_RDTSC_EXIT;
5663 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5664 AssertRC(rc);
5665 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5666 }
5667 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5668 }
5669}
5670
5671
5672/**
5673 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5674 * VM-exit interruption info type.
5675 *
5676 * @returns The IEM exception flags.
5677 * @param uVector The event vector.
5678 * @param uVmxVectorType The VMX event type.
5679 *
5680 * @remarks This function currently only constructs flags required for
5681 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5682 * and CR2 aspects of an exception are not included).
5683 */
5684static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5685{
5686 uint32_t fIemXcptFlags;
5687 switch (uVmxVectorType)
5688 {
5689 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5690 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5691 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5692 break;
5693
5694 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5695 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5696 break;
5697
5698 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5699 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5700 break;
5701
5702 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5703 {
5704 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5705 if (uVector == X86_XCPT_BP)
5706 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5707 else if (uVector == X86_XCPT_OF)
5708 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5709 else
5710 {
5711 fIemXcptFlags = 0;
5712 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5713 }
5714 break;
5715 }
5716
5717 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5718 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5719 break;
5720
5721 default:
5722 fIemXcptFlags = 0;
5723 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5724 break;
5725 }
5726 return fIemXcptFlags;
5727}
5728
5729
5730/**
5731 * Sets an event as a pending event to be injected into the guest.
5732 *
5733 * @param pVCpu The cross context virtual CPU structure.
5734 * @param u32IntInfo The VM-entry interruption-information field.
5735 * @param cbInstr The VM-entry instruction length in bytes (for software
5736 * interrupts, exceptions and privileged software
5737 * exceptions).
5738 * @param u32ErrCode The VM-entry exception error code.
5739 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5740 * page-fault.
5741 *
5742 * @remarks Statistics counter assumes this is a guest event being injected or
5743 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5744 * always incremented.
5745 */
5746DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5747 RTGCUINTPTR GCPtrFaultAddress)
5748{
5749 Assert(!pVCpu->hm.s.Event.fPending);
5750 pVCpu->hm.s.Event.fPending = true;
5751 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5752 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5753 pVCpu->hm.s.Event.cbInstr = cbInstr;
5754 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5755}
5756
5757
5758/**
5759 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5760 *
5761 * @param pVCpu The cross context virtual CPU structure.
5762 */
5763DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
5764{
5765 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
5766 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5767 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5768 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5769 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5770}
5771
5772
5773/**
5774 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
5775 *
5776 * @param pVCpu The cross context virtual CPU structure.
5777 */
5778DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
5779{
5780 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
5781 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5782 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
5783 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5784 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5785}
5786
5787
5788/**
5789 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
5790 *
5791 * @param pVCpu The cross context virtual CPU structure.
5792 */
5793DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
5794{
5795 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
5796 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5797 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
5798 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5799 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5800}
5801
5802
5803#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5804/**
5805 * Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
5806 *
5807 * @param pVCpu The cross context virtual CPU structure.
5808 * @param u32ErrCode The error code for the general-protection exception.
5809 */
5810DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
5811{
5812 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
5813 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5814 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5815 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5816 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrCode, 0 /* GCPtrFaultAddress */);
5817}
5818
5819
5820/**
5821 * Sets a stack (\#SS) exception as pending-for-injection into the VM.
5822 *
5823 * @param pVCpu The cross context virtual CPU structure.
5824 * @param u32ErrCode The error code for the stack exception.
5825 */
5826DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
5827{
5828 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
5829 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5830 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5831 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5832 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrCode, 0 /* GCPtrFaultAddress */);
5833}
5834
5835
5836/**
5837 * Decodes the memory operand of an instruction that caused a VM-exit.
5838 *
5839 * The VM-exit qualification field provides the displacement field for memory
5840 * operand instructions, if any.
5841 *
5842 * @returns Strict VBox status code (i.e. informational status codes too).
5843 * @retval VINF_SUCCESS if the operand was successfully decoded.
5844 * @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
5845 * operand.
5846 * @param pVCpu The cross context virtual CPU structure.
5847 * @param uExitInstrInfo The VM-exit instruction information field.
5848 * @param enmMemAccess The memory operand's access type (read or write).
5849 * @param GCPtrDisp The instruction displacement field, if any. For
5850 * RIP-relative addressing pass RIP + displacement here.
5851 * @param pGCPtrMem Where to store the effective destination memory address.
5852 */
5853static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
5854 PRTGCPTR pGCPtrMem)
5855{
5856 Assert(pGCPtrMem);
5857 Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
5858 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_EFER | CPUMCTX_EXTRN_CR0);
5859
5860 static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
5861 static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
5862 AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
5863
5864 VMXEXITINSTRINFO ExitInstrInfo;
5865 ExitInstrInfo.u = uExitInstrInfo;
5866 uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
5867 uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
5868 bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
5869 uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
5870 uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
5871 bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
5872 uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
5873 bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
5874 bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
5875
5876 /*
5877 * Validate instruction information.
5878 * This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
5879 */
5880 AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
5881 ("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
5882 AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
5883 ("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
5884 AssertLogRelMsgReturn(fIsMemOperand,
5885 ("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
5886
5887 /*
5888 * Compute the complete effective address.
5889 *
5890 * See AMD instruction spec. 1.4.2 "SIB Byte Format"
5891 * See AMD spec. 4.5.2 "Segment Registers".
5892 */
5893 RTGCPTR GCPtrMem = GCPtrDisp;
5894 if (fBaseRegValid)
5895 GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
5896 if (fIdxRegValid)
5897 GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
5898
5899 RTGCPTR const GCPtrOff = GCPtrMem;
5900 if ( !fIsLongMode
5901 || iSegReg >= X86_SREG_FS)
5902 GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
5903 GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
5904
5905 /*
5906 * Validate effective address.
5907 * See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
5908 */
5909 uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
5910 Assert(cbAccess > 0);
5911 if (fIsLongMode)
5912 {
5913 if (X86_IS_CANONICAL(GCPtrMem))
5914 {
5915 *pGCPtrMem = GCPtrMem;
5916 return VINF_SUCCESS;
5917 }
5918
5919 /** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
5920 * "Data Limit Checks in 64-bit Mode". */
5921 Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
5922 hmR0VmxSetPendingXcptGP(pVCpu, 0);
5923 return VINF_HM_PENDING_XCPT;
5924 }
5925
5926 /*
5927 * This is a watered down version of iemMemApplySegment().
5928 * Parts that are not applicable for VMX instructions like real-or-v8086 mode
5929 * and segment CPL/DPL checks are skipped.
5930 */
5931 RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
5932 RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
5933 PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
5934
5935 /* Check if the segment is present and usable. */
5936 if ( pSel->Attr.n.u1Present
5937 && !pSel->Attr.n.u1Unusable)
5938 {
5939 Assert(pSel->Attr.n.u1DescType);
5940 if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
5941 {
5942 /* Check permissions for the data segment. */
5943 if ( enmMemAccess == VMXMEMACCESS_WRITE
5944 && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
5945 {
5946 Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
5947 hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
5948 return VINF_HM_PENDING_XCPT;
5949 }
5950
5951 /* Check limits if it's a normal data segment. */
5952 if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
5953 {
5954 if ( GCPtrFirst32 > pSel->u32Limit
5955 || GCPtrLast32 > pSel->u32Limit)
5956 {
5957 Log4Func(("Data segment limit exceeded."
5958 "iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
5959 GCPtrLast32, pSel->u32Limit));
5960 if (iSegReg == X86_SREG_SS)
5961 hmR0VmxSetPendingXcptSS(pVCpu, 0);
5962 else
5963 hmR0VmxSetPendingXcptGP(pVCpu, 0);
5964 return VINF_HM_PENDING_XCPT;
5965 }
5966 }
5967 else
5968 {
5969 /* Check limits if it's an expand-down data segment.
5970 Note! The upper boundary is defined by the B bit, not the G bit! */
5971 if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
5972 || GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
5973 {
5974 Log4Func(("Expand-down data segment limit exceeded."
5975 "iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
5976 GCPtrLast32, pSel->u32Limit));
5977 if (iSegReg == X86_SREG_SS)
5978 hmR0VmxSetPendingXcptSS(pVCpu, 0);
5979 else
5980 hmR0VmxSetPendingXcptGP(pVCpu, 0);
5981 return VINF_HM_PENDING_XCPT;
5982 }
5983 }
5984 }
5985 else
5986 {
5987 /* Check permissions for the code segment. */
5988 if ( enmMemAccess == VMXMEMACCESS_WRITE
5989 || ( enmMemAccess == VMXMEMACCESS_READ
5990 && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
5991 {
5992 Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
5993 Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
5994 hmR0VmxSetPendingXcptGP(pVCpu, 0);
5995 return VINF_HM_PENDING_XCPT;
5996 }
5997
5998 /* Check limits for the code segment (normal/expand-down not applicable for code segments). */
5999 if ( GCPtrFirst32 > pSel->u32Limit
6000 || GCPtrLast32 > pSel->u32Limit)
6001 {
6002 Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
6003 GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
6004 if (iSegReg == X86_SREG_SS)
6005 hmR0VmxSetPendingXcptSS(pVCpu, 0);
6006 else
6007 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6008 return VINF_HM_PENDING_XCPT;
6009 }
6010 }
6011 }
6012 else
6013 {
6014 Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6015 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6016 return VINF_HM_PENDING_XCPT;
6017 }
6018
6019 *pGCPtrMem = GCPtrMem;
6020 return VINF_SUCCESS;
6021}
6022
6023
6024/**
6025 * Perform the relevant VMX instruction checks for VM-exits that occurred due to the
6026 * guest attempting to execute a VMX instruction.
6027 *
6028 * @returns Strict VBox status code (i.e. informational status codes too).
6029 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
6030 * @retval VINF_HM_PENDING_XCPT if an exception was raised.
6031 *
6032 * @param pVCpu The cross context virtual CPU structure.
6033 * @param uExitReason The VM-exit reason.
6034 *
6035 * @todo NstVmx: Document other error codes when VM-exit is implemented.
6036 * @remarks No-long-jump zone!!!
6037 */
6038static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
6039{
6040 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS
6041 | CPUMCTX_EXTRN_HWVIRT);
6042
6043 if ( CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx)
6044 || ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6045 && !CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx)))
6046 {
6047 Log4Func(("In real/v86-mode or long-mode outside 64-bit code segment -> #UD\n"));
6048 hmR0VmxSetPendingXcptUD(pVCpu);
6049 return VINF_HM_PENDING_XCPT;
6050 }
6051
6052 if (uExitReason == VMX_EXIT_VMXON)
6053 {
6054 /*
6055 * We check CR4.VMXE because it is required to be always set while in VMX operation
6056 * by physical CPUs and our CR4 read shadow is only consulted when executing specific
6057 * instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
6058 * otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
6059 */
6060 if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
6061 {
6062 Log4Func(("CR4.VMXE is not set -> #UD\n"));
6063 hmR0VmxSetPendingXcptUD(pVCpu);
6064 return VINF_HM_PENDING_XCPT;
6065 }
6066 }
6067 else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
6068 {
6069 /*
6070 * The guest has not entered VMX operation but attempted to execute a VMX instruction
6071 * (other than VMXON), we need to raise a #UD.
6072 */
6073 Log4Func(("Not in VMX root mode -> #UD\n"));
6074 hmR0VmxSetPendingXcptUD(pVCpu);
6075 return VINF_HM_PENDING_XCPT;
6076 }
6077
6078 if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
6079 {
6080 /*
6081 * The nested-guest attempted to execute a VMX instruction, cause a VM-exit and let
6082 * the guest hypervisor deal with it.
6083 */
6084 /** @todo NSTVMX: Trigger a VM-exit */
6085 }
6086
6087 /*
6088 * VMX instructions require CPL 0 except in VMX non-root mode where the VM-exit intercept
6089 * (above) takes preceedence over the CPL check.
6090 */
6091 if (CPUMGetGuestCPL(pVCpu) > 0)
6092 {
6093 Log4Func(("CPL > 0 -> #GP(0)\n"));
6094 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6095 return VINF_HM_PENDING_XCPT;
6096 }
6097
6098 return VINF_SUCCESS;
6099}
6100
6101#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
6102
6103
6104/**
6105 * Handle a condition that occurred while delivering an event through the guest
6106 * IDT.
6107 *
6108 * @returns Strict VBox status code (i.e. informational status codes too).
6109 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
6110 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
6111 * to continue execution of the guest which will delivery the \#DF.
6112 * @retval VINF_EM_RESET if we detected a triple-fault condition.
6113 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
6114 *
6115 * @param pVCpu The cross context virtual CPU structure.
6116 * @param pVmxTransient Pointer to the VMX transient structure.
6117 *
6118 * @remarks No-long-jump zone!!!
6119 */
6120static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6121{
6122 uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
6123
6124 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
6125 rc2 |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
6126 AssertRCReturn(rc2, rc2);
6127
6128 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
6129 if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6130 {
6131 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
6132 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
6133
6134 /*
6135 * If the event was a software interrupt (generated with INT n) or a software exception
6136 * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
6137 * can handle the VM-exit and continue guest execution which will re-execute the
6138 * instruction rather than re-injecting the exception, as that can cause premature
6139 * trips to ring-3 before injection and involve TRPM which currently has no way of
6140 * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
6141 * the problem).
6142 */
6143 IEMXCPTRAISE enmRaise;
6144 IEMXCPTRAISEINFO fRaiseInfo;
6145 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6146 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6147 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6148 {
6149 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
6150 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6151 }
6152 else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
6153 {
6154 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
6155 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
6156 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
6157 /** @todo Make AssertMsgReturn as just AssertMsg later. */
6158 AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
6159 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
6160 uExitVectorType), VERR_VMX_IPE_5);
6161
6162 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
6163
6164 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
6165 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
6166 {
6167 pVmxTransient->fVectoringPF = true;
6168 enmRaise = IEMXCPTRAISE_PREV_EVENT;
6169 }
6170 }
6171 else
6172 {
6173 /*
6174 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
6175 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
6176 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
6177 */
6178 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6179 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6180 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
6181 enmRaise = IEMXCPTRAISE_PREV_EVENT;
6182 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6183 }
6184
6185 /*
6186 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
6187 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
6188 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
6189 * subsequent VM-entry would fail.
6190 *
6191 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
6192 */
6193 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
6194 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6195 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
6196 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
6197 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
6198 {
6199 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6200 }
6201
6202 switch (enmRaise)
6203 {
6204 case IEMXCPTRAISE_CURRENT_XCPT:
6205 {
6206 Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
6207 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
6208 Assert(rcStrict == VINF_SUCCESS);
6209 break;
6210 }
6211
6212 case IEMXCPTRAISE_PREV_EVENT:
6213 {
6214 uint32_t u32ErrCode;
6215 if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
6216 {
6217 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6218 AssertRCReturn(rc2, rc2);
6219 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6220 }
6221 else
6222 u32ErrCode = 0;
6223
6224 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
6225 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6226 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6227 0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
6228
6229 Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
6230 pVCpu->hm.s.Event.u32ErrCode));
6231 Assert(rcStrict == VINF_SUCCESS);
6232 break;
6233 }
6234
6235 case IEMXCPTRAISE_REEXEC_INSTR:
6236 Assert(rcStrict == VINF_SUCCESS);
6237 break;
6238
6239 case IEMXCPTRAISE_DOUBLE_FAULT:
6240 {
6241 /*
6242 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
6243 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
6244 */
6245 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6246 {
6247 pVmxTransient->fVectoringDoublePF = true;
6248 Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
6249 pVCpu->cpum.GstCtx.cr2));
6250 rcStrict = VINF_SUCCESS;
6251 }
6252 else
6253 {
6254 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6255 hmR0VmxSetPendingXcptDF(pVCpu);
6256 Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
6257 uIdtVector, uExitVector));
6258 rcStrict = VINF_HM_DOUBLE_FAULT;
6259 }
6260 break;
6261 }
6262
6263 case IEMXCPTRAISE_TRIPLE_FAULT:
6264 {
6265 Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
6266 rcStrict = VINF_EM_RESET;
6267 break;
6268 }
6269
6270 case IEMXCPTRAISE_CPU_HANG:
6271 {
6272 Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
6273 rcStrict = VERR_EM_GUEST_CPU_HANG;
6274 break;
6275 }
6276
6277 default:
6278 {
6279 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6280 rcStrict = VERR_VMX_IPE_2;
6281 break;
6282 }
6283 }
6284 }
6285 else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6286 && VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6287 && uExitVector != X86_XCPT_DF
6288 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
6289 {
6290 /*
6291 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6292 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6293 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6294 */
6295 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6296 {
6297 Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
6298 VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6299 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6300 }
6301 }
6302
6303 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
6304 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
6305 return rcStrict;
6306}
6307
6308
6309/**
6310 * Imports a guest segment register from the current VMCS into
6311 * the guest-CPU context.
6312 *
6313 * @returns VBox status code.
6314 * @param pVCpu The cross context virtual CPU structure.
6315 * @param idxSel Index of the selector in the VMCS.
6316 * @param idxLimit Index of the segment limit in the VMCS.
6317 * @param idxBase Index of the segment base in the VMCS.
6318 * @param idxAccess Index of the access rights of the segment in the VMCS.
6319 * @param pSelReg Pointer to the segment selector.
6320 *
6321 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6322 * do not log!
6323 *
6324 * @remarks Never call this function directly!!! Use the
6325 * HMVMX_IMPORT_SREG() macro as that takes care
6326 * of whether to read from the VMCS cache or not.
6327 */
6328static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6329 PCPUMSELREG pSelReg)
6330{
6331 NOREF(pVCpu);
6332
6333 uint32_t u32Sel;
6334 uint32_t u32Limit;
6335 uint32_t u32Attr;
6336 uint64_t u64Base;
6337 int rc = VMXReadVmcs32(idxSel, &u32Sel);
6338 rc |= VMXReadVmcs32(idxLimit, &u32Limit);
6339 rc |= VMXReadVmcs32(idxAccess, &u32Attr);
6340 rc |= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
6341 AssertRCReturn(rc, rc);
6342
6343 pSelReg->Sel = (uint16_t)u32Sel;
6344 pSelReg->ValidSel = (uint16_t)u32Sel;
6345 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6346 pSelReg->u32Limit = u32Limit;
6347 pSelReg->u64Base = u64Base;
6348 pSelReg->Attr.u = u32Attr;
6349
6350 /*
6351 * If VT-x marks the segment as unusable, most other bits remain undefined:
6352 * - For CS the L, D and G bits have meaning.
6353 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6354 * - For the remaining data segments no bits are defined.
6355 *
6356 * The present bit and the unusable bit has been observed to be set at the
6357 * same time (the selector was supposed to be invalid as we started executing
6358 * a V8086 interrupt in ring-0).
6359 *
6360 * What should be important for the rest of the VBox code, is that the P bit is
6361 * cleared. Some of the other VBox code recognizes the unusable bit, but
6362 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6363 * safe side here, we'll strip off P and other bits we don't care about. If
6364 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6365 *
6366 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6367 */
6368 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6369 {
6370 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6371
6372 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6373 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6374 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6375#ifdef VBOX_STRICT
6376 VMMRZCallRing3Disable(pVCpu);
6377 Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
6378# ifdef DEBUG_bird
6379 AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
6380 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6381 idxSel, u32Sel, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6382# endif
6383 VMMRZCallRing3Enable(pVCpu);
6384#endif
6385 }
6386 return VINF_SUCCESS;
6387}
6388
6389
6390/**
6391 * Imports the guest RIP from the VMCS back into the guest-CPU context.
6392 *
6393 * @returns VBox status code.
6394 * @param pVCpu The cross context virtual CPU structure.
6395 *
6396 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6397 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6398 * instead!!!
6399 */
6400DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6401{
6402 uint64_t u64Val;
6403 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6404 if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6405 {
6406 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6407 if (RT_SUCCESS(rc))
6408 {
6409 pCtx->rip = u64Val;
6410 EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
6411 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6412 }
6413 return rc;
6414 }
6415 return VINF_SUCCESS;
6416}
6417
6418
6419/**
6420 * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6421 *
6422 * @returns VBox status code.
6423 * @param pVCpu The cross context virtual CPU structure.
6424 *
6425 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6426 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6427 * instead!!!
6428 */
6429DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6430{
6431 uint32_t u32Val;
6432 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6433 if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6434 {
6435 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6436 if (RT_SUCCESS(rc))
6437 {
6438 pCtx->eflags.u32 = u32Val;
6439
6440 /* Restore eflags for real-on-v86-mode hack. */
6441 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6442 {
6443 pCtx->eflags.Bits.u1VM = 0;
6444 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6445 }
6446 }
6447 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6448 return rc;
6449 }
6450 return VINF_SUCCESS;
6451}
6452
6453
6454/**
6455 * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6456 * context.
6457 *
6458 * @returns VBox status code.
6459 * @param pVCpu The cross context virtual CPU structure.
6460 *
6461 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6462 * do not log!
6463 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6464 * instead!!!
6465 */
6466DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6467{
6468 uint32_t u32Val;
6469 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6470 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
6471 if (RT_SUCCESS(rc))
6472 {
6473 /*
6474 * We additionally have a requirement to import RIP, RFLAGS depending on whether we
6475 * might need them in hmR0VmxEvaluatePendingEvent().
6476 */
6477 if (!u32Val)
6478 {
6479 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6480 {
6481 rc = hmR0VmxImportGuestRip(pVCpu);
6482 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6483 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6484 }
6485
6486 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6487 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6488 }
6489 else
6490 {
6491 rc = hmR0VmxImportGuestRip(pVCpu);
6492 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6493
6494 if (u32Val & ( VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
6495 | VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
6496 {
6497 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6498 }
6499 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6500 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6501
6502 if (u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
6503 {
6504 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6505 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6506 }
6507 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6508 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6509 }
6510 }
6511 return rc;
6512}
6513
6514
6515/**
6516 * Worker for VMXR0ImportStateOnDemand.
6517 *
6518 * @returns VBox status code.
6519 * @param pVCpu The cross context virtual CPU structure.
6520 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6521 */
6522static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6523{
6524#define VMXLOCAL_BREAK_RC(a_rc) \
6525 if (RT_FAILURE(a_rc)) \
6526 break
6527
6528 int rc = VINF_SUCCESS;
6529 PVM pVM = pVCpu->CTX_SUFF(pVM);
6530 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6531 uint64_t u64Val;
6532 uint32_t u32Val;
6533
6534 Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6535 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6536
6537 /*
6538 * We disable interrupts to make the updating of the state and in particular
6539 * the fExtrn modification atomic wrt to preemption hooks.
6540 */
6541 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6542
6543 fWhat &= pCtx->fExtrn;
6544 if (fWhat)
6545 {
6546 do
6547 {
6548 if (fWhat & CPUMCTX_EXTRN_RIP)
6549 {
6550 rc = hmR0VmxImportGuestRip(pVCpu);
6551 VMXLOCAL_BREAK_RC(rc);
6552 }
6553
6554 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6555 {
6556 rc = hmR0VmxImportGuestRFlags(pVCpu);
6557 VMXLOCAL_BREAK_RC(rc);
6558 }
6559
6560 if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6561 {
6562 rc = hmR0VmxImportGuestIntrState(pVCpu);
6563 VMXLOCAL_BREAK_RC(rc);
6564 }
6565
6566 if (fWhat & CPUMCTX_EXTRN_RSP)
6567 {
6568 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6569 VMXLOCAL_BREAK_RC(rc);
6570 pCtx->rsp = u64Val;
6571 }
6572
6573 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6574 {
6575 if (fWhat & CPUMCTX_EXTRN_CS)
6576 {
6577 rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6578 rc |= hmR0VmxImportGuestRip(pVCpu);
6579 VMXLOCAL_BREAK_RC(rc);
6580 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6581 pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6582 EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true);
6583 }
6584 if (fWhat & CPUMCTX_EXTRN_SS)
6585 {
6586 rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6587 VMXLOCAL_BREAK_RC(rc);
6588 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6589 pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6590 }
6591 if (fWhat & CPUMCTX_EXTRN_DS)
6592 {
6593 rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6594 VMXLOCAL_BREAK_RC(rc);
6595 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6596 pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6597 }
6598 if (fWhat & CPUMCTX_EXTRN_ES)
6599 {
6600 rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6601 VMXLOCAL_BREAK_RC(rc);
6602 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6603 pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6604 }
6605 if (fWhat & CPUMCTX_EXTRN_FS)
6606 {
6607 rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6608 VMXLOCAL_BREAK_RC(rc);
6609 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6610 pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6611 }
6612 if (fWhat & CPUMCTX_EXTRN_GS)
6613 {
6614 rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6615 VMXLOCAL_BREAK_RC(rc);
6616 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6617 pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6618 }
6619 }
6620
6621 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6622 {
6623 if (fWhat & CPUMCTX_EXTRN_LDTR)
6624 {
6625 rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6626 VMXLOCAL_BREAK_RC(rc);
6627 }
6628
6629 if (fWhat & CPUMCTX_EXTRN_GDTR)
6630 {
6631 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6632 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6633 VMXLOCAL_BREAK_RC(rc);
6634 pCtx->gdtr.pGdt = u64Val;
6635 pCtx->gdtr.cbGdt = u32Val;
6636 }
6637
6638 /* Guest IDTR. */
6639 if (fWhat & CPUMCTX_EXTRN_IDTR)
6640 {
6641 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6642 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6643 VMXLOCAL_BREAK_RC(rc);
6644 pCtx->idtr.pIdt = u64Val;
6645 pCtx->idtr.cbIdt = u32Val;
6646 }
6647
6648 /* Guest TR. */
6649 if (fWhat & CPUMCTX_EXTRN_TR)
6650 {
6651 /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6652 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6653 {
6654 rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6655 VMXLOCAL_BREAK_RC(rc);
6656 }
6657 }
6658 }
6659
6660 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6661 {
6662 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6663 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6664 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6665 pCtx->SysEnter.cs = u32Val;
6666 VMXLOCAL_BREAK_RC(rc);
6667 }
6668
6669#if HC_ARCH_BITS == 64
6670 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6671 {
6672 if ( pVM->hm.s.fAllow64BitGuests
6673 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6674 pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6675 }
6676
6677 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6678 {
6679 if ( pVM->hm.s.fAllow64BitGuests
6680 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6681 {
6682 pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6683 pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6684 pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6685 }
6686 }
6687#endif
6688
6689 if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
6690#if HC_ARCH_BITS == 32
6691 || (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS))
6692#endif
6693 )
6694 {
6695 PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6696 uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6697 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6698 {
6699 switch (pMsr->u32Msr)
6700 {
6701#if HC_ARCH_BITS == 32
6702 case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6703 case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6704 case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6705 case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6706#endif
6707 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6708 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6709 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit */ break;
6710 default:
6711 {
6712 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6713 ASMSetFlags(fEFlags);
6714 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6715 cMsrs));
6716 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6717 }
6718 }
6719 }
6720 }
6721
6722 if (fWhat & CPUMCTX_EXTRN_DR7)
6723 {
6724 if (!pVCpu->hm.s.fUsingHyperDR7)
6725 {
6726 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6727 rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6728 VMXLOCAL_BREAK_RC(rc);
6729 pCtx->dr[7] = u32Val;
6730 }
6731 }
6732
6733 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6734 {
6735 uint32_t u32Shadow;
6736 if (fWhat & CPUMCTX_EXTRN_CR0)
6737 {
6738 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6739 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6740 VMXLOCAL_BREAK_RC(rc);
6741 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr0Mask)
6742 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr0Mask);
6743 VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6744 CPUMSetGuestCR0(pVCpu, u32Val);
6745 VMMRZCallRing3Enable(pVCpu);
6746 }
6747
6748 if (fWhat & CPUMCTX_EXTRN_CR4)
6749 {
6750 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6751 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6752 VMXLOCAL_BREAK_RC(rc);
6753 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr4Mask)
6754 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr4Mask);
6755 CPUMSetGuestCR4(pVCpu, u32Val);
6756 }
6757
6758 if (fWhat & CPUMCTX_EXTRN_CR3)
6759 {
6760 /* CR0.PG bit changes are always intercepted, so it's up to date. */
6761 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6762 || ( pVM->hm.s.fNestedPaging
6763 && CPUMIsGuestPagingEnabledEx(pCtx)))
6764 {
6765 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6766 if (pCtx->cr3 != u64Val)
6767 {
6768 CPUMSetGuestCR3(pVCpu, u64Val);
6769 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6770 }
6771
6772 /* If the guest is in PAE mode, sync back the PDPE's into the guest state.
6773 Note: CR4.PAE, CR0.PG, EFER bit changes are always intercepted, so they're up to date. */
6774 if (CPUMIsGuestInPAEModeEx(pCtx))
6775 {
6776 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6777 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6778 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6779 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6780 VMXLOCAL_BREAK_RC(rc);
6781 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6782 }
6783 }
6784 }
6785 }
6786 } while (0);
6787
6788 if (RT_SUCCESS(rc))
6789 {
6790 /* Update fExtrn. */
6791 pCtx->fExtrn &= ~fWhat;
6792
6793 /* If everything has been imported, clear the HM keeper bit. */
6794 if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6795 {
6796 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6797 Assert(!pCtx->fExtrn);
6798 }
6799 }
6800 }
6801 else
6802 AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6803
6804 ASMSetFlags(fEFlags);
6805
6806 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6807
6808 /*
6809 * Honor any pending CR3 updates.
6810 *
6811 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6812 * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6813 * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6814 *
6815 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6816 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6817 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6818 * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6819 *
6820 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6821 */
6822 if (VMMRZCallRing3IsEnabled(pVCpu))
6823 {
6824 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6825 {
6826 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
6827 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6828 }
6829
6830 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6831 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6832
6833 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6834 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6835 }
6836
6837 return VINF_SUCCESS;
6838#undef VMXLOCAL_BREAK_RC
6839}
6840
6841
6842/**
6843 * Saves the guest state from the VMCS into the guest-CPU context.
6844 *
6845 * @returns VBox status code.
6846 * @param pVCpu The cross context virtual CPU structure.
6847 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6848 */
6849VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6850{
6851 return hmR0VmxImportGuestState(pVCpu, fWhat);
6852}
6853
6854
6855/**
6856 * Check per-VM and per-VCPU force flag actions that require us to go back to
6857 * ring-3 for one reason or another.
6858 *
6859 * @returns Strict VBox status code (i.e. informational status codes too)
6860 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6861 * ring-3.
6862 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6863 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6864 * interrupts)
6865 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6866 * all EMTs to be in ring-3.
6867 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6868 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6869 * to the EM loop.
6870 *
6871 * @param pVCpu The cross context virtual CPU structure.
6872 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6873 */
6874static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
6875{
6876 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6877
6878 /*
6879 * Anything pending? Should be more likely than not if we're doing a good job.
6880 */
6881 PVM pVM = pVCpu->CTX_SUFF(pVM);
6882 if ( !fStepping
6883 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6884 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6885 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6886 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6887 return VINF_SUCCESS;
6888
6889 /* Pending PGM C3 sync. */
6890 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6891 {
6892 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6893 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
6894 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
6895 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6896 if (rcStrict2 != VINF_SUCCESS)
6897 {
6898 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6899 Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6900 return rcStrict2;
6901 }
6902 }
6903
6904 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6905 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6906 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6907 {
6908 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6909 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6910 Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6911 return rc2;
6912 }
6913
6914 /* Pending VM request packets, such as hardware interrupts. */
6915 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6916 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6917 {
6918 Log4Func(("Pending VM request forcing us back to ring-3\n"));
6919 return VINF_EM_PENDING_REQUEST;
6920 }
6921
6922 /* Pending PGM pool flushes. */
6923 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6924 {
6925 Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6926 return VINF_PGM_POOL_FLUSH_PENDING;
6927 }
6928
6929 /* Pending DMA requests. */
6930 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6931 {
6932 Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6933 return VINF_EM_RAW_TO_R3;
6934 }
6935
6936 return VINF_SUCCESS;
6937}
6938
6939
6940/**
6941 * Converts any TRPM trap into a pending HM event. This is typically used when
6942 * entering from ring-3 (not longjmp returns).
6943 *
6944 * @param pVCpu The cross context virtual CPU structure.
6945 */
6946static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6947{
6948 Assert(TRPMHasTrap(pVCpu));
6949 Assert(!pVCpu->hm.s.Event.fPending);
6950
6951 uint8_t uVector;
6952 TRPMEVENT enmTrpmEvent;
6953 RTGCUINT uErrCode;
6954 RTGCUINTPTR GCPtrFaultAddress;
6955 uint8_t cbInstr;
6956
6957 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6958 AssertRC(rc);
6959
6960 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6961 uint32_t u32IntInfo = uVector | VMX_EXIT_INT_INFO_VALID;
6962 if (enmTrpmEvent == TRPM_TRAP)
6963 {
6964 switch (uVector)
6965 {
6966 case X86_XCPT_NMI:
6967 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6968 break;
6969
6970 case X86_XCPT_BP:
6971 case X86_XCPT_OF:
6972 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6973 break;
6974
6975 case X86_XCPT_PF:
6976 case X86_XCPT_DF:
6977 case X86_XCPT_TS:
6978 case X86_XCPT_NP:
6979 case X86_XCPT_SS:
6980 case X86_XCPT_GP:
6981 case X86_XCPT_AC:
6982 u32IntInfo |= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
6983 RT_FALL_THRU();
6984 default:
6985 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6986 break;
6987 }
6988 }
6989 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6990 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6991 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6992 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6993 else
6994 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6995
6996 rc = TRPMResetTrap(pVCpu);
6997 AssertRC(rc);
6998 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6999 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7000
7001 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7002}
7003
7004
7005/**
7006 * Converts the pending HM event into a TRPM trap.
7007 *
7008 * @param pVCpu The cross context virtual CPU structure.
7009 */
7010static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7011{
7012 Assert(pVCpu->hm.s.Event.fPending);
7013
7014 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7015 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7016 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
7017 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7018
7019 /* If a trap was already pending, we did something wrong! */
7020 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7021
7022 TRPMEVENT enmTrapType;
7023 switch (uVectorType)
7024 {
7025 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7026 enmTrapType = TRPM_HARDWARE_INT;
7027 break;
7028
7029 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7030 enmTrapType = TRPM_SOFTWARE_INT;
7031 break;
7032
7033 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7034 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7035 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7036 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7037 enmTrapType = TRPM_TRAP;
7038 break;
7039
7040 default:
7041 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7042 enmTrapType = TRPM_32BIT_HACK;
7043 break;
7044 }
7045
7046 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7047
7048 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7049 AssertRC(rc);
7050
7051 if (fErrorCodeValid)
7052 TRPMSetErrorCode(pVCpu, uErrorCode);
7053
7054 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7055 && uVector == X86_XCPT_PF)
7056 {
7057 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7058 }
7059 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7060 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7061 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7062 {
7063 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7064 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
7065 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7066 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7067 }
7068
7069 /* Clear any pending events from the VMCS. */
7070 VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7071 VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, 0);
7072
7073 /* We're now done converting the pending event. */
7074 pVCpu->hm.s.Event.fPending = false;
7075}
7076
7077
7078/**
7079 * Does the necessary state syncing before returning to ring-3 for any reason
7080 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7081 *
7082 * @returns VBox status code.
7083 * @param pVCpu The cross context virtual CPU structure.
7084 * @param fImportState Whether to import the guest state from the VMCS back
7085 * to the guest-CPU context.
7086 *
7087 * @remarks No-long-jmp zone!!!
7088 */
7089static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
7090{
7091 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7092 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7093
7094 RTCPUID idCpu = RTMpCpuId();
7095 Log4Func(("HostCpuId=%u\n", idCpu));
7096
7097 /*
7098 * !!! IMPORTANT !!!
7099 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7100 */
7101
7102 /* Save the guest state if necessary. */
7103 if (fImportState)
7104 {
7105 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
7106 AssertRCReturn(rc, rc);
7107 }
7108
7109 /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
7110 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7111 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
7112
7113 /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
7114#ifdef VBOX_STRICT
7115 if (CPUMIsHyperDebugStateActive(pVCpu))
7116 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
7117#endif
7118 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7119 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7120 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7121
7122#if HC_ARCH_BITS == 64
7123 /* Restore host-state bits that VT-x only restores partially. */
7124 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7125 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7126 {
7127 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7128 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7129 }
7130 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7131#endif
7132
7133 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7134 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7135 {
7136 /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
7137 if (!fImportState)
7138 {
7139 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS);
7140 AssertRCReturn(rc, rc);
7141 }
7142 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7143 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7144 }
7145 else
7146 pVCpu->hm.s.vmx.fLazyMsrs = 0;
7147
7148 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7149 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7150
7151 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7152 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
7153 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
7154 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
7155 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
7156 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7157 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7158 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7159 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7160
7161 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7162
7163 /** @todo This partially defeats the purpose of having preemption hooks.
7164 * The problem is, deregistering the hooks should be moved to a place that
7165 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7166 * context.
7167 */
7168 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7169 {
7170 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7171 AssertRCReturn(rc, rc);
7172
7173 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7174 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7175 }
7176 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7177 NOREF(idCpu);
7178
7179 return VINF_SUCCESS;
7180}
7181
7182
7183/**
7184 * Leaves the VT-x session.
7185 *
7186 * @returns VBox status code.
7187 * @param pVCpu The cross context virtual CPU structure.
7188 *
7189 * @remarks No-long-jmp zone!!!
7190 */
7191static int hmR0VmxLeaveSession(PVMCPU pVCpu)
7192{
7193 HM_DISABLE_PREEMPT(pVCpu);
7194 HMVMX_ASSERT_CPU_SAFE(pVCpu);
7195 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7196 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7197
7198 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7199 and done this from the VMXR0ThreadCtxCallback(). */
7200 if (!pVCpu->hm.s.fLeaveDone)
7201 {
7202 int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
7203 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7204 pVCpu->hm.s.fLeaveDone = true;
7205 }
7206 Assert(!pVCpu->cpum.GstCtx.fExtrn);
7207
7208 /*
7209 * !!! IMPORTANT !!!
7210 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7211 */
7212
7213 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7214 /** @todo Deregistering here means we need to VMCLEAR always
7215 * (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
7216 * for calling VMMR0ThreadCtxHookDisable here! */
7217 VMMR0ThreadCtxHookDisable(pVCpu);
7218
7219 /* Leave HM context. This takes care of local init (term). */
7220 int rc = HMR0LeaveCpu(pVCpu);
7221
7222 HM_RESTORE_PREEMPT();
7223 return rc;
7224}
7225
7226
7227/**
7228 * Does the necessary state syncing before doing a longjmp to ring-3.
7229 *
7230 * @returns VBox status code.
7231 * @param pVCpu The cross context virtual CPU structure.
7232 *
7233 * @remarks No-long-jmp zone!!!
7234 */
7235DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
7236{
7237 return hmR0VmxLeaveSession(pVCpu);
7238}
7239
7240
7241/**
7242 * Take necessary actions before going back to ring-3.
7243 *
7244 * An action requires us to go back to ring-3. This function does the necessary
7245 * steps before we can safely return to ring-3. This is not the same as longjmps
7246 * to ring-3, this is voluntary and prepares the guest so it may continue
7247 * executing outside HM (recompiler/IEM).
7248 *
7249 * @returns VBox status code.
7250 * @param pVCpu The cross context virtual CPU structure.
7251 * @param rcExit The reason for exiting to ring-3. Can be
7252 * VINF_VMM_UNKNOWN_RING3_CALL.
7253 */
7254static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
7255{
7256 Assert(pVCpu);
7257 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7258
7259 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7260 {
7261 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7262 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7263 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7264 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7265 }
7266
7267 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7268 VMMRZCallRing3Disable(pVCpu);
7269 Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
7270
7271 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7272 if (pVCpu->hm.s.Event.fPending)
7273 {
7274 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7275 Assert(!pVCpu->hm.s.Event.fPending);
7276 }
7277
7278 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7279 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7280
7281 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7282 and if we're injecting an event we should have a TRPM trap pending. */
7283 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7284#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
7285 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7286#endif
7287
7288 /* Save guest state and restore host state bits. */
7289 int rc = hmR0VmxLeaveSession(pVCpu);
7290 AssertRCReturn(rc, rc);
7291 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7292 /* Thread-context hooks are unregistered at this point!!! */
7293
7294 /* Sync recompiler state. */
7295 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7296 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7297 | CPUM_CHANGED_LDTR
7298 | CPUM_CHANGED_GDTR
7299 | CPUM_CHANGED_IDTR
7300 | CPUM_CHANGED_TR
7301 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7302 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
7303 && CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
7304 {
7305 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7306 }
7307
7308 Assert(!pVCpu->hm.s.fClearTrapFlag);
7309
7310 /* Update the exit-to-ring 3 reason. */
7311 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
7312
7313 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7314 if (rcExit != VINF_EM_RAW_INTERRUPT)
7315 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7316
7317 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7318
7319 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7320 VMMRZCallRing3RemoveNotification(pVCpu);
7321 VMMRZCallRing3Enable(pVCpu);
7322
7323 return rc;
7324}
7325
7326
7327/**
7328 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7329 * longjump to ring-3 and possibly get preempted.
7330 *
7331 * @returns VBox status code.
7332 * @param pVCpu The cross context virtual CPU structure.
7333 * @param enmOperation The operation causing the ring-3 longjump.
7334 * @param pvUser User argument, currently unused, NULL.
7335 */
7336static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7337{
7338 RT_NOREF(pvUser);
7339 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7340 {
7341 /*
7342 * !!! IMPORTANT !!!
7343 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7344 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7345 */
7346 VMMRZCallRing3RemoveNotification(pVCpu);
7347 VMMRZCallRing3Disable(pVCpu);
7348 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7349 RTThreadPreemptDisable(&PreemptState);
7350
7351 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
7352 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7353 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7354
7355#if HC_ARCH_BITS == 64
7356 /* Restore host-state bits that VT-x only restores partially. */
7357 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7358 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7359 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7360 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7361#endif
7362
7363 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7364 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7365 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7366
7367 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7368 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7369 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7370 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7371 {
7372 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7373 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7374 }
7375
7376 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7377 VMMR0ThreadCtxHookDisable(pVCpu);
7378 HMR0LeaveCpu(pVCpu);
7379 RTThreadPreemptRestore(&PreemptState);
7380 return VINF_SUCCESS;
7381 }
7382
7383 Assert(pVCpu);
7384 Assert(pvUser);
7385 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7386 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7387
7388 VMMRZCallRing3Disable(pVCpu);
7389 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7390
7391 Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7392
7393 int rc = hmR0VmxLongJmpToRing3(pVCpu);
7394 AssertRCReturn(rc, rc);
7395
7396 VMMRZCallRing3Enable(pVCpu);
7397 return VINF_SUCCESS;
7398}
7399
7400
7401/**
7402 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7403 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7404 *
7405 * @param pVCpu The cross context virtual CPU structure.
7406 */
7407DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7408{
7409 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7410 {
7411 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7412 {
7413 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_PROC_CTLS_INT_WINDOW_EXIT;
7414 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7415 AssertRC(rc);
7416 Log4Func(("Setup interrupt-window exiting\n"));
7417 }
7418 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7419}
7420
7421
7422/**
7423 * Clears the interrupt-window exiting control in the VMCS.
7424 *
7425 * @param pVCpu The cross context virtual CPU structure.
7426 */
7427DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7428{
7429 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT);
7430 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
7431 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7432 AssertRC(rc);
7433 Log4Func(("Cleared interrupt-window exiting\n"));
7434}
7435
7436
7437/**
7438 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7439 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7440 *
7441 * @param pVCpu The cross context virtual CPU structure.
7442 */
7443DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7444{
7445 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7446 {
7447 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7448 {
7449 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7450 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7451 AssertRC(rc);
7452 Log4Func(("Setup NMI-window exiting\n"));
7453 }
7454 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7455}
7456
7457
7458/**
7459 * Clears the NMI-window exiting control in the VMCS.
7460 *
7461 * @param pVCpu The cross context virtual CPU structure.
7462 */
7463DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7464{
7465 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT);
7466 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7467 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7468 AssertRC(rc);
7469 Log4Func(("Cleared NMI-window exiting\n"));
7470}
7471
7472
7473/**
7474 * Evaluates the event to be delivered to the guest and sets it as the pending
7475 * event.
7476 *
7477 * @returns The VT-x guest-interruptibility state.
7478 * @param pVCpu The cross context virtual CPU structure.
7479 */
7480static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu)
7481{
7482 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7483 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7484 uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu);
7485 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7486 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7487 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7488
7489 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7490 Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7491 Assert(!fBlockSti || pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7492 Assert(!TRPMHasTrap(pVCpu));
7493
7494 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7495 APICUpdatePendingInterrupts(pVCpu);
7496
7497 /*
7498 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7499 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7500 */
7501 /** @todo SMI. SMIs take priority over NMIs. */
7502 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7503 {
7504 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7505 if ( !pVCpu->hm.s.Event.fPending
7506 && !fBlockNmi
7507 && !fBlockSti
7508 && !fBlockMovSS)
7509 {
7510 Log4Func(("Pending NMI\n"));
7511 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INT_INFO_VALID;
7512 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7513
7514 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7515 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7516 }
7517 else
7518 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7519 }
7520 /*
7521 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7522 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7523 */
7524 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7525 && !pVCpu->hm.s.fSingleInstruction)
7526 {
7527 Assert(!DBGFIsStepping(pVCpu));
7528 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7529 AssertRCReturn(rc, 0);
7530 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7531 if ( !pVCpu->hm.s.Event.fPending
7532 && !fBlockInt
7533 && !fBlockSti
7534 && !fBlockMovSS)
7535 {
7536 uint8_t u8Interrupt;
7537 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7538 if (RT_SUCCESS(rc))
7539 {
7540 Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7541 uint32_t u32IntInfo = u8Interrupt
7542 | VMX_EXIT_INT_INFO_VALID
7543 | (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7544
7545 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7546 }
7547 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7548 {
7549 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
7550 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7551 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7552
7553 /*
7554 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7555 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7556 * need to re-set this force-flag here.
7557 */
7558 }
7559 else
7560 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7561 }
7562 else
7563 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7564 }
7565
7566 return fIntrState;
7567}
7568
7569
7570/**
7571 * Sets a pending-debug exception to be delivered to the guest if the guest is
7572 * single-stepping in the VMCS.
7573 *
7574 * @param pVCpu The cross context virtual CPU structure.
7575 */
7576DECLINLINE(int) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7577{
7578 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7579 RT_NOREF(pVCpu);
7580 return VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
7581}
7582
7583
7584/**
7585 * Injects any pending events into the guest if the guest is in a state to
7586 * receive them.
7587 *
7588 * @returns Strict VBox status code (i.e. informational status codes too).
7589 * @param pVCpu The cross context virtual CPU structure.
7590 * @param fIntrState The VT-x guest-interruptibility state.
7591 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7592 * return VINF_EM_DBG_STEPPED if the event was
7593 * dispatched directly.
7594 */
7595static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, uint32_t fIntrState, bool fStepping)
7596{
7597 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7598 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7599
7600 bool fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7601 bool fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7602
7603 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7604 Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7605 Assert(!fBlockSti || pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7606 Assert(!TRPMHasTrap(pVCpu));
7607
7608 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7609 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7610 if (pVCpu->hm.s.Event.fPending)
7611 {
7612 /*
7613 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7614 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7615 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7616 *
7617 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7618 */
7619 uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7620#ifdef VBOX_STRICT
7621 if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
7622 {
7623 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7624 Assert(!fBlockInt);
7625 Assert(!fBlockSti);
7626 Assert(!fBlockMovSS);
7627 }
7628 else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
7629 {
7630 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7631 Assert(!fBlockSti);
7632 Assert(!fBlockMovSS);
7633 Assert(!fBlockNmi);
7634 }
7635#endif
7636 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7637 uIntType));
7638 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7639 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7640 &fIntrState);
7641 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7642
7643 /* Update the interruptibility-state as it could have been changed by
7644 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7645 fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7646 fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7647
7648 if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
7649 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7650 else
7651 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7652 }
7653
7654 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7655 if ( fBlockSti
7656 || fBlockMovSS)
7657 {
7658 if (!pVCpu->hm.s.fSingleInstruction)
7659 {
7660 /*
7661 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7662 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7663 * See Intel spec. 27.3.4 "Saving Non-Register State".
7664 */
7665 Assert(!DBGFIsStepping(pVCpu));
7666 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7667 AssertRCReturn(rc, rc);
7668 if (pCtx->eflags.Bits.u1TF)
7669 {
7670 int rc2 = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7671 AssertRCReturn(rc2, rc2);
7672 }
7673 }
7674 else if (pCtx->eflags.Bits.u1TF)
7675 {
7676 /*
7677 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7678 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7679 */
7680 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
7681 fIntrState = 0;
7682 }
7683 }
7684
7685 /*
7686 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7687 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7688 */
7689 int rc3 = hmR0VmxExportGuestIntrState(pVCpu, fIntrState);
7690 AssertRCReturn(rc3, rc3);
7691
7692 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7693 NOREF(fBlockMovSS); NOREF(fBlockSti);
7694 return rcStrict;
7695}
7696
7697
7698/**
7699 * Injects a double-fault (\#DF) exception into the VM.
7700 *
7701 * @returns Strict VBox status code (i.e. informational status codes too).
7702 * @param pVCpu The cross context virtual CPU structure.
7703 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7704 * and should return VINF_EM_DBG_STEPPED if the event
7705 * is injected directly (register modified by us, not
7706 * by hardware on VM-entry).
7707 * @param pfIntrState Pointer to the current guest interruptibility-state.
7708 * This interruptibility-state will be updated if
7709 * necessary. This cannot not be NULL.
7710 */
7711DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, bool fStepping, uint32_t *pfIntrState)
7712{
7713 uint32_t const u32IntInfo = X86_XCPT_DF | VMX_EXIT_INT_INFO_VALID
7714 | (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT)
7715 | VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7716 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */, fStepping,
7717 pfIntrState);
7718}
7719
7720
7721/**
7722 * Injects a general-protection (\#GP) fault into the VM.
7723 *
7724 * @returns Strict VBox status code (i.e. informational status codes too).
7725 * @param pVCpu The cross context virtual CPU structure.
7726 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7727 * mode, i.e. in real-mode it's not valid).
7728 * @param u32ErrorCode The error code associated with the \#GP.
7729 * @param fStepping Whether we're running in
7730 * hmR0VmxRunGuestCodeStep() and should return
7731 * VINF_EM_DBG_STEPPED if the event is injected
7732 * directly (register modified by us, not by
7733 * hardware on VM-entry).
7734 * @param pfIntrState Pointer to the current guest interruptibility-state.
7735 * This interruptibility-state will be updated if
7736 * necessary. This cannot not be NULL.
7737 */
7738DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, bool fErrorCodeValid, uint32_t u32ErrorCode, bool fStepping,
7739 uint32_t *pfIntrState)
7740{
7741 uint32_t const u32IntInfo = X86_XCPT_GP | VMX_EXIT_INT_INFO_VALID
7742 | (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT)
7743 | (fErrorCodeValid ? VMX_EXIT_INT_INFO_ERROR_CODE_VALID : 0);
7744 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */, fStepping,
7745 pfIntrState);
7746}
7747
7748
7749/**
7750 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7751 * stack.
7752 *
7753 * @returns Strict VBox status code (i.e. informational status codes too).
7754 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7755 * @param pVCpu The cross context virtual CPU structure.
7756 * @param uValue The value to push to the guest stack.
7757 */
7758static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
7759{
7760 /*
7761 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7762 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7763 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7764 */
7765 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7766 if (pCtx->sp == 1)
7767 return VINF_EM_RESET;
7768 pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7769 int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
7770 AssertRC(rc);
7771 return rc;
7772}
7773
7774
7775/**
7776 * Injects an event into the guest upon VM-entry by updating the relevant fields
7777 * in the VM-entry area in the VMCS.
7778 *
7779 * @returns Strict VBox status code (i.e. informational status codes too).
7780 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7781 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7782 *
7783 * @param pVCpu The cross context virtual CPU structure.
7784 * @param u64IntInfo The VM-entry interruption-information field.
7785 * @param cbInstr The VM-entry instruction length in bytes (for
7786 * software interrupts, exceptions and privileged
7787 * software exceptions).
7788 * @param u32ErrCode The VM-entry exception error code.
7789 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7790 * @param pfIntrState Pointer to the current guest interruptibility-state.
7791 * This interruptibility-state will be updated if
7792 * necessary. This cannot not be NULL.
7793 * @param fStepping Whether we're running in
7794 * hmR0VmxRunGuestCodeStep() and should return
7795 * VINF_EM_DBG_STEPPED if the event is injected
7796 * directly (register modified by us, not by
7797 * hardware on VM-entry).
7798 */
7799static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7800 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7801{
7802 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7803 AssertMsg(!RT_HI_U32(u64IntInfo), ("%#RX64\n", u64IntInfo));
7804 Assert(pfIntrState);
7805
7806 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7807 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7808 uint32_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
7809 uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
7810
7811#ifdef VBOX_STRICT
7812 /*
7813 * Validate the error-code-valid bit for hardware exceptions.
7814 * No error codes for exceptions in real-mode.
7815 *
7816 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7817 */
7818 if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
7819 && !CPUMIsGuestInRealModeEx(pCtx))
7820 {
7821 switch (uVector)
7822 {
7823 case X86_XCPT_PF:
7824 case X86_XCPT_DF:
7825 case X86_XCPT_TS:
7826 case X86_XCPT_NP:
7827 case X86_XCPT_SS:
7828 case X86_XCPT_GP:
7829 case X86_XCPT_AC:
7830 AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
7831 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7832 RT_FALL_THRU();
7833 default:
7834 break;
7835 }
7836 }
7837#endif
7838
7839 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7840 Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
7841 || !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
7842
7843 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7844
7845 /*
7846 * Hardware interrupts & exceptions cannot be delivered through the software interrupt
7847 * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7848 * interrupt handler in the (real-mode) guest.
7849 *
7850 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7851 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7852 */
7853 if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
7854 {
7855 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7856 {
7857 /*
7858 * For unrestricted execution enabled CPUs running real-mode guests, we must not
7859 * set the deliver-error-code bit.
7860 *
7861 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7862 */
7863 u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
7864 }
7865 else
7866 {
7867 PVM pVM = pVCpu->CTX_SUFF(pVM);
7868 Assert(PDMVmmDevHeapIsEnabled(pVM));
7869 Assert(pVM->hm.s.vmx.pRealModeTSS);
7870
7871 /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7872 int rc2 = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_TABLE_MASK | CPUMCTX_EXTRN_RIP
7873 | CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_RFLAGS);
7874 AssertRCReturn(rc2, rc2);
7875
7876 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7877 size_t const cbIdtEntry = sizeof(X86IDTR16);
7878 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
7879 {
7880 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7881 if (uVector == X86_XCPT_DF)
7882 return VINF_EM_RESET;
7883
7884 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7885 if (uVector == X86_XCPT_GP)
7886 return hmR0VmxInjectXcptDF(pVCpu, fStepping, pfIntrState);
7887
7888 /*
7889 * If we're injecting an event with no valid IDT entry, inject a #GP.
7890 * No error codes for exceptions in real-mode.
7891 *
7892 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7893 */
7894 return hmR0VmxInjectXcptGP(pVCpu, false /* fErrCodeValid */, 0 /* u32ErrCode */, fStepping, pfIntrState);
7895 }
7896
7897 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7898 uint16_t uGuestIp = pCtx->ip;
7899 if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
7900 {
7901 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7902 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7903 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7904 }
7905 else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
7906 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7907
7908 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7909 X86IDTR16 IdtEntry;
7910 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
7911 rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7912 AssertRCReturn(rc2, rc2);
7913
7914 /* Construct the stack frame for the interrupt/exception handler. */
7915 VBOXSTRICTRC rcStrict;
7916 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
7917 if (rcStrict == VINF_SUCCESS)
7918 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
7919 if (rcStrict == VINF_SUCCESS)
7920 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
7921
7922 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7923 if (rcStrict == VINF_SUCCESS)
7924 {
7925 pCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7926 pCtx->rip = IdtEntry.offSel;
7927 pCtx->cs.Sel = IdtEntry.uSel;
7928 pCtx->cs.ValidSel = IdtEntry.uSel;
7929 pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7930 if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
7931 && uVector == X86_XCPT_PF)
7932 pCtx->cr2 = GCPtrFaultAddress;
7933
7934 /* If any other guest-state bits are changed here, make sure to update
7935 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7936 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS | HM_CHANGED_GUEST_CR2
7937 | HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
7938 | HM_CHANGED_GUEST_RSP);
7939
7940 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7941 if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
7942 {
7943 Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
7944 && uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
7945 Log4Func(("Clearing inhibition due to STI\n"));
7946 *pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
7947 }
7948 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7949 u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
7950
7951 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7952 it, if we are returning to ring-3 before executing guest code. */
7953 pVCpu->hm.s.Event.fPending = false;
7954
7955 /* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
7956 if (fStepping)
7957 rcStrict = VINF_EM_DBG_STEPPED;
7958 }
7959 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7960 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7961 return rcStrict;
7962 }
7963 }
7964
7965 /* Validate. */
7966 Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7967 Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
7968
7969 /* Inject. */
7970 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7971 if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
7972 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7973 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7974 AssertRCReturn(rc, rc);
7975
7976 /* Update CR2. */
7977 if ( VMX_ENTRY_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
7978 && uVector == X86_XCPT_PF)
7979 pCtx->cr2 = GCPtrFaultAddress;
7980
7981 Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
7982
7983 return VINF_SUCCESS;
7984}
7985
7986
7987/**
7988 * Clears the interrupt-window exiting control in the VMCS and if necessary
7989 * clears the current event in the VMCS as well.
7990 *
7991 * @returns VBox status code.
7992 * @param pVCpu The cross context virtual CPU structure.
7993 *
7994 * @remarks Use this function only to clear events that have not yet been
7995 * delivered to the guest but are injected in the VMCS!
7996 * @remarks No-long-jump zone!!!
7997 */
7998static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
7999{
8000 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8001 {
8002 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8003 Log4Func(("Cleared interrupt window\n"));
8004 }
8005
8006 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8007 {
8008 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8009 Log4Func(("Cleared NMI window\n"));
8010 }
8011}
8012
8013
8014/**
8015 * Enters the VT-x session.
8016 *
8017 * @returns VBox status code.
8018 * @param pVCpu The cross context virtual CPU structure.
8019 * @param pHostCpu Pointer to the global CPU info struct.
8020 */
8021VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu, PHMGLOBALCPUINFO pHostCpu)
8022{
8023 AssertPtr(pVCpu);
8024 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8025 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8026 RT_NOREF(pHostCpu);
8027
8028 LogFlowFunc(("pVCpu=%p\n", pVCpu));
8029 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8030 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8031
8032#ifdef VBOX_STRICT
8033 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8034 RTCCUINTREG uHostCR4 = ASMGetCR4();
8035 if (!(uHostCR4 & X86_CR4_VMXE))
8036 {
8037 LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8038 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8039 }
8040#endif
8041
8042 /*
8043 * Load the VCPU's VMCS as the current (and active) one.
8044 */
8045 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8046 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8047 if (RT_FAILURE(rc))
8048 return rc;
8049
8050 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8051 pVCpu->hm.s.fLeaveDone = false;
8052 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8053
8054 return VINF_SUCCESS;
8055}
8056
8057
8058/**
8059 * The thread-context callback (only on platforms which support it).
8060 *
8061 * @param enmEvent The thread-context event.
8062 * @param pVCpu The cross context virtual CPU structure.
8063 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8064 * @thread EMT(pVCpu)
8065 */
8066VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8067{
8068 NOREF(fGlobalInit);
8069
8070 switch (enmEvent)
8071 {
8072 case RTTHREADCTXEVENT_OUT:
8073 {
8074 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8075 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8076 VMCPU_ASSERT_EMT(pVCpu);
8077
8078 /* No longjmps (logger flushes, locks) in this fragile context. */
8079 VMMRZCallRing3Disable(pVCpu);
8080 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8081
8082 /*
8083 * Restore host-state (FPU, debug etc.)
8084 */
8085 if (!pVCpu->hm.s.fLeaveDone)
8086 {
8087 /*
8088 * Do -not- import the guest-state here as we might already be in the middle of importing
8089 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
8090 */
8091 hmR0VmxLeave(pVCpu, false /* fImportState */);
8092 pVCpu->hm.s.fLeaveDone = true;
8093 }
8094
8095 /* Leave HM context, takes care of local init (term). */
8096 int rc = HMR0LeaveCpu(pVCpu);
8097 AssertRC(rc); NOREF(rc);
8098
8099 /* Restore longjmp state. */
8100 VMMRZCallRing3Enable(pVCpu);
8101 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8102 break;
8103 }
8104
8105 case RTTHREADCTXEVENT_IN:
8106 {
8107 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8108 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8109 VMCPU_ASSERT_EMT(pVCpu);
8110
8111 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8112 VMMRZCallRing3Disable(pVCpu);
8113 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8114
8115 /* Initialize the bare minimum state required for HM. This takes care of
8116 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8117 int rc = hmR0EnterCpu(pVCpu);
8118 AssertRC(rc);
8119 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8120 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8121
8122 /* Load the active VMCS as the current one. */
8123 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8124 {
8125 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8126 AssertRC(rc); NOREF(rc);
8127 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8128 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8129 }
8130 pVCpu->hm.s.fLeaveDone = false;
8131
8132 /* Restore longjmp state. */
8133 VMMRZCallRing3Enable(pVCpu);
8134 break;
8135 }
8136
8137 default:
8138 break;
8139 }
8140}
8141
8142
8143/**
8144 * Exports the host state into the VMCS host-state area.
8145 * Sets up the VM-exit MSR-load area.
8146 *
8147 * The CPU state will be loaded from these fields on every successful VM-exit.
8148 *
8149 * @returns VBox status code.
8150 * @param pVCpu The cross context virtual CPU structure.
8151 *
8152 * @remarks No-long-jump zone!!!
8153 */
8154static int hmR0VmxExportHostState(PVMCPU pVCpu)
8155{
8156 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8157
8158 int rc = VINF_SUCCESS;
8159 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8160 {
8161 rc = hmR0VmxExportHostControlRegs();
8162 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8163
8164 rc = hmR0VmxExportHostSegmentRegs(pVCpu);
8165 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8166
8167 rc = hmR0VmxExportHostMsrs(pVCpu);
8168 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8169
8170 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
8171 }
8172 return rc;
8173}
8174
8175
8176/**
8177 * Saves the host state in the VMCS host-state.
8178 *
8179 * @returns VBox status code.
8180 * @param pVCpu The cross context virtual CPU structure.
8181 *
8182 * @remarks No-long-jump zone!!!
8183 */
8184VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
8185{
8186 AssertPtr(pVCpu);
8187 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8188
8189 /*
8190 * Export the host state here while entering HM context.
8191 * When thread-context hooks are used, we might get preempted and have to re-save the host
8192 * state but most of the time we won't be, so do it here before we disable interrupts.
8193 */
8194 return hmR0VmxExportHostState(pVCpu);
8195}
8196
8197
8198/**
8199 * Exports the guest state into the VMCS guest-state area.
8200 *
8201 * The will typically be done before VM-entry when the guest-CPU state and the
8202 * VMCS state may potentially be out of sync.
8203 *
8204 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8205 * VM-entry controls.
8206 * Sets up the appropriate VMX non-root function to execute guest code based on
8207 * the guest CPU mode.
8208 *
8209 * @returns VBox strict status code.
8210 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8211 * without unrestricted guest access and the VMMDev is not presently
8212 * mapped (e.g. EFI32).
8213 *
8214 * @param pVCpu The cross context virtual CPU structure.
8215 *
8216 * @remarks No-long-jump zone!!!
8217 */
8218static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu)
8219{
8220 AssertPtr(pVCpu);
8221 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8222
8223 LogFlowFunc(("pVCpu=%p\n", pVCpu));
8224
8225 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
8226
8227 /* Determine real-on-v86 mode. */
8228 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8229 if ( !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
8230 && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
8231 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8232
8233 /*
8234 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8235 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8236 */
8237 int rc = hmR0VmxSelectVMRunHandler(pVCpu);
8238 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8239
8240 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8241 rc = hmR0VmxExportGuestEntryCtls(pVCpu);
8242 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8243
8244 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8245 rc = hmR0VmxExportGuestExitCtls(pVCpu);
8246 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8247
8248 rc = hmR0VmxExportGuestCR0(pVCpu);
8249 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8250
8251 VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu);
8252 if (rcStrict == VINF_SUCCESS)
8253 { /* likely */ }
8254 else
8255 {
8256 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8257 return rcStrict;
8258 }
8259
8260 rc = hmR0VmxExportGuestSegmentRegs(pVCpu);
8261 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8262
8263 /* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8264 may alter controls if we determine we don't have to swap EFER after all. */
8265 rc = hmR0VmxExportGuestMsrs(pVCpu);
8266 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8267
8268 rc = hmR0VmxExportGuestApicTpr(pVCpu);
8269 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8270
8271 rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8272 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8273
8274 /* Exporting RFLAGS here is fine, even though RFLAGS.TF might depend on guest debug state which is
8275 not exported here. It is re-evaluated and updated if necessary in hmR0VmxExportSharedState(). */
8276 rc = hmR0VmxExportGuestRip(pVCpu);
8277 rc |= hmR0VmxExportGuestRsp(pVCpu);
8278 rc |= hmR0VmxExportGuestRflags(pVCpu);
8279 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8280
8281 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8282 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8283 | HM_CHANGED_GUEST_CR2
8284 | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8285 | HM_CHANGED_GUEST_X87
8286 | HM_CHANGED_GUEST_SSE_AVX
8287 | HM_CHANGED_GUEST_OTHER_XSAVE
8288 | HM_CHANGED_GUEST_XCRx
8289 | HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8290 | HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8291 | HM_CHANGED_GUEST_TSC_AUX
8292 | HM_CHANGED_GUEST_OTHER_MSRS
8293 | HM_CHANGED_GUEST_HWVIRT
8294 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8295
8296 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8297 return rc;
8298}
8299
8300
8301/**
8302 * Exports the state shared between the host and guest into the VMCS.
8303 *
8304 * @param pVCpu The cross context virtual CPU structure.
8305 *
8306 * @remarks No-long-jump zone!!!
8307 */
8308static void hmR0VmxExportSharedState(PVMCPU pVCpu)
8309{
8310 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8311 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8312
8313 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8314 {
8315 int rc = hmR0VmxExportSharedDebugState(pVCpu);
8316 AssertRC(rc);
8317 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8318
8319 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8320 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8321 {
8322 rc = hmR0VmxExportGuestRflags(pVCpu);
8323 AssertRC(rc);
8324 }
8325 }
8326
8327 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8328 {
8329 hmR0VmxLazyLoadGuestMsrs(pVCpu);
8330 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8331 }
8332
8333 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8334 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8335}
8336
8337
8338/**
8339 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8340 *
8341 * @returns Strict VBox status code (i.e. informational status codes too).
8342 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8343 * without unrestricted guest access and the VMMDev is not presently
8344 * mapped (e.g. EFI32).
8345 *
8346 * @param pVCpu The cross context virtual CPU structure.
8347 *
8348 * @remarks No-long-jump zone!!!
8349 */
8350static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu)
8351{
8352 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8353 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8354 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8355
8356#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8357 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8358#endif
8359
8360 /*
8361 * For many exits it's only RIP that changes and hence try to export it first
8362 * without going through a lot of change flag checks.
8363 */
8364 VBOXSTRICTRC rcStrict;
8365 uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8366 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8367 if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8368 {
8369 rcStrict = hmR0VmxExportGuestRip(pVCpu);
8370 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8371 { /* likely */}
8372 else
8373 AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8374 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8375 }
8376 else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8377 {
8378 rcStrict = hmR0VmxExportGuestState(pVCpu);
8379 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8380 { /* likely */}
8381 else
8382 {
8383 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8384 VBOXSTRICTRC_VAL(rcStrict)));
8385 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8386 return rcStrict;
8387 }
8388 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8389 }
8390 else
8391 rcStrict = VINF_SUCCESS;
8392
8393#ifdef VBOX_STRICT
8394 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8395 fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8396 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8397 AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8398 ("fCtxChanged=%#RX64\n", fCtxChanged));
8399#endif
8400 return rcStrict;
8401}
8402
8403
8404/**
8405 * Does the preparations before executing guest code in VT-x.
8406 *
8407 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8408 * recompiler/IEM. We must be cautious what we do here regarding committing
8409 * guest-state information into the VMCS assuming we assuredly execute the
8410 * guest in VT-x mode.
8411 *
8412 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8413 * the common-state (TRPM/forceflags), we must undo those changes so that the
8414 * recompiler/IEM can (and should) use them when it resumes guest execution.
8415 * Otherwise such operations must be done when we can no longer exit to ring-3.
8416 *
8417 * @returns Strict VBox status code (i.e. informational status codes too).
8418 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8419 * have been disabled.
8420 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8421 * double-fault into the guest.
8422 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8423 * dispatched directly.
8424 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8425 *
8426 * @param pVCpu The cross context virtual CPU structure.
8427 * @param pVmxTransient Pointer to the VMX transient structure.
8428 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8429 * us ignore some of the reasons for returning to
8430 * ring-3, and return VINF_EM_DBG_STEPPED if event
8431 * dispatching took place.
8432 */
8433static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
8434{
8435 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8436
8437#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_ONLY_IN_IEM
8438 Log2(("hmR0SvmPreRunGuest: Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
8439 return VINF_EM_RESCHEDULE_REM;
8440#endif
8441
8442#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8443 PGMRZDynMapFlushAutoSet(pVCpu);
8444#endif
8445
8446 /* Check force flag actions that might require us to go back to ring-3. */
8447 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
8448 if (rcStrict == VINF_SUCCESS)
8449 { /* FFs doesn't get set all the time. */ }
8450 else
8451 return rcStrict;
8452
8453 /*
8454 * Setup the virtualized-APIC accesses.
8455 *
8456 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8457 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8458 *
8459 * This is the reason we do it here and not in hmR0VmxExportGuestState().
8460 */
8461 PVM pVM = pVCpu->CTX_SUFF(pVM);
8462 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8463 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
8464 && PDMHasApic(pVM))
8465 {
8466 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8467 Assert(u64MsrApicBase);
8468 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8469
8470 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8471
8472 /* Unalias any existing mapping. */
8473 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8474 AssertRCReturn(rc, rc);
8475
8476 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8477 Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8478 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8479 AssertRCReturn(rc, rc);
8480
8481 /* Update the per-VCPU cache of the APIC base MSR. */
8482 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8483 }
8484
8485 if (TRPMHasTrap(pVCpu))
8486 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8487 uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu);
8488
8489 /*
8490 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
8491 * needs to be done with longjmps or interrupts + preemption enabled. Event injection might
8492 * also result in triple-faulting the VM.
8493 */
8494 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, fIntrState, fStepping);
8495 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8496 { /* likely */ }
8497 else
8498 {
8499 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8500 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8501 return rcStrict;
8502 }
8503
8504 /*
8505 * A longjump might result in importing CR3 even for VM-exits that don't necessarily
8506 * import CR3 themselves. We will need to update them here, as even as late as the above
8507 * hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
8508 * the below force flags to be set.
8509 */
8510 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8511 {
8512 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
8513 int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8514 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
8515 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
8516 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8517 }
8518 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8519 {
8520 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8521 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8522 }
8523
8524 /*
8525 * No longjmps to ring-3 from this point on!!!
8526 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8527 * This also disables flushing of the R0-logger instance (if any).
8528 */
8529 VMMRZCallRing3Disable(pVCpu);
8530
8531 /*
8532 * Export the guest state bits.
8533 *
8534 * We cannot perform longjmps while loading the guest state because we do not preserve the
8535 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8536 * CPU migration.
8537 *
8538 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8539 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8540 * Hence, loading of the guest state needs to be done -after- injection of events.
8541 */
8542 rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu);
8543 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8544 { /* likely */ }
8545 else
8546 {
8547 VMMRZCallRing3Enable(pVCpu);
8548 return rcStrict;
8549 }
8550
8551 /*
8552 * We disable interrupts so that we don't miss any interrupts that would flag preemption
8553 * (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
8554 * preemption disabled for a while. Since this is purly to aid the
8555 * RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
8556 * disable interrupt on NT.
8557 *
8558 * We need to check for force-flags that could've possible been altered since we last
8559 * checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
8560 * see @bugref{6398}).
8561 *
8562 * We also check a couple of other force-flags as a last opportunity to get the EMT back
8563 * to ring-3 before executing guest code.
8564 */
8565 pVmxTransient->fEFlags = ASMIntDisableFlags();
8566
8567 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8568 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8569 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8570 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8571 {
8572 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8573 {
8574 pVCpu->hm.s.Event.fPending = false;
8575
8576 /*
8577 * We've injected any pending events. This is really the point of no return (to ring-3).
8578 *
8579 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8580 * returns from this function, so don't enable them here.
8581 */
8582 return VINF_SUCCESS;
8583 }
8584
8585 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
8586 rcStrict = VINF_EM_RAW_INTERRUPT;
8587 }
8588 else
8589 {
8590 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8591 rcStrict = VINF_EM_RAW_TO_R3;
8592 }
8593
8594 ASMSetFlags(pVmxTransient->fEFlags);
8595 VMMRZCallRing3Enable(pVCpu);
8596
8597 return rcStrict;
8598}
8599
8600
8601/**
8602 * Prepares to run guest code in VT-x and we've committed to doing so. This
8603 * means there is no backing out to ring-3 or anywhere else at this
8604 * point.
8605 *
8606 * @param pVCpu The cross context virtual CPU structure.
8607 * @param pVmxTransient Pointer to the VMX transient structure.
8608 *
8609 * @remarks Called with preemption disabled.
8610 * @remarks No-long-jump zone!!!
8611 */
8612static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
8613{
8614 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8615 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8616 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8617
8618 /*
8619 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8620 */
8621 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8622 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8623
8624 PVM pVM = pVCpu->CTX_SUFF(pVM);
8625 if (!CPUMIsGuestFPUStateActive(pVCpu))
8626 {
8627 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8628 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8629 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
8630 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8631 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8632 }
8633
8634 /*
8635 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8636 */
8637 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8638 && pVCpu->hm.s.vmx.cMsrs > 0)
8639 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8640
8641 /*
8642 * Re-save the host state bits as we may've been preempted (only happens when
8643 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8644 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8645 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8646 * See @bugref{8432}.
8647 */
8648 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8649 {
8650 int rc = hmR0VmxExportHostState(pVCpu);
8651 AssertRC(rc);
8652 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8653 }
8654 Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8655
8656 /*
8657 * Export the state shared between host and guest (FPU, debug, lazy MSRs).
8658 */
8659 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8660 hmR0VmxExportSharedState(pVCpu);
8661 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8662
8663 /* Store status of the shared guest-host state at the time of VM-entry. */
8664#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8665 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
8666 {
8667 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8668 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8669 }
8670 else
8671#endif
8672 {
8673 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8674 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8675 }
8676
8677 /*
8678 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8679 */
8680 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
8681 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8682
8683 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8684 RTCPUID idCurrentCpu = pCpu->idCpu;
8685 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8686 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8687 {
8688 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8689 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8690 }
8691
8692 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8693 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8694 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8695 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8696
8697 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8698
8699 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8700 to start executing. */
8701
8702 /*
8703 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8704 */
8705 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
8706 {
8707 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
8708 {
8709 bool fMsrUpdated;
8710 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8711 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8712 &fMsrUpdated);
8713 AssertRC(rc2);
8714 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8715 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8716 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8717 }
8718 else
8719 {
8720 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8721 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8722 }
8723 }
8724
8725 if (pVM->cpum.ro.GuestFeatures.fIbrs)
8726 {
8727 bool fMsrUpdated;
8728 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8729 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8730 &fMsrUpdated);
8731 AssertRC(rc2);
8732 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8733 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8734 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8735 }
8736
8737#ifdef VBOX_STRICT
8738 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8739 hmR0VmxCheckHostEferMsr(pVCpu);
8740 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8741#endif
8742#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8743 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
8744 {
8745 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
8746 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8747 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8748 }
8749#endif
8750}
8751
8752
8753/**
8754 * Performs some essential restoration of state after running guest code in
8755 * VT-x.
8756 *
8757 * @param pVCpu The cross context virtual CPU structure.
8758 * @param pVmxTransient Pointer to the VMX transient structure.
8759 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8760 *
8761 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8762 *
8763 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8764 * unconditionally when it is safe to do so.
8765 */
8766static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8767{
8768 uint64_t const uHostTsc = ASMReadTSC();
8769 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8770
8771 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8772 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8773 pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8774 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8775 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8776 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8777
8778 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
8779 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TscOffset);
8780
8781 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
8782 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8783 Assert(!ASMIntAreEnabled());
8784 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8785
8786#if HC_ARCH_BITS == 64
8787 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8788#endif
8789#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8790 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8791 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8792 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8793#else
8794 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8795#endif
8796#ifdef VBOX_STRICT
8797 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8798#endif
8799 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8800
8801 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8802 uint32_t uExitReason;
8803 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8804 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8805 AssertRC(rc);
8806 pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
8807 pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
8808
8809 if (rcVMRun == VINF_SUCCESS)
8810 {
8811 /*
8812 * Update the VM-exit history array here even if the VM-entry failed due to:
8813 * - Invalid guest state.
8814 * - MSR loading.
8815 * - Machine-check event.
8816 *
8817 * In any of the above cases we will still have a "valid" VM-exit reason
8818 * despite @a fVMEntryFailed being false.
8819 *
8820 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8821 *
8822 * Note! We don't have CS or RIP at this point. Will probably address that later
8823 * by amending the history entry added here.
8824 */
8825 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8826 UINT64_MAX, uHostTsc);
8827
8828 if (!pVmxTransient->fVMEntryFailed)
8829 {
8830 VMMRZCallRing3Enable(pVCpu);
8831
8832 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8833 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8834
8835#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8836 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8837 AssertRC(rc);
8838#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8839 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8840 AssertRC(rc);
8841#else
8842 /*
8843 * Import the guest-interruptibility state always as we need it while evaluating
8844 * injecting events on re-entry.
8845 *
8846 * We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8847 * checking for real-mode while exporting the state because all bits that cause
8848 * mode changes wrt CR0 are intercepted.
8849 */
8850 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8851 AssertRC(rc);
8852#endif
8853
8854 /*
8855 * Sync the TPR shadow with our APIC state.
8856 */
8857 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
8858 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8859 {
8860 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8861 AssertRC(rc);
8862 ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8863 }
8864
8865 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8866 return;
8867 }
8868 }
8869 else
8870 Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8871
8872 VMMRZCallRing3Enable(pVCpu);
8873}
8874
8875
8876/**
8877 * Runs the guest code using VT-x the normal way.
8878 *
8879 * @returns VBox status code.
8880 * @param pVCpu The cross context virtual CPU structure.
8881 *
8882 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8883 */
8884static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu)
8885{
8886 VMXTRANSIENT VmxTransient;
8887 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8888 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8889 uint32_t cLoops = 0;
8890
8891 for (;; cLoops++)
8892 {
8893 Assert(!HMR0SuspendPending());
8894 HMVMX_ASSERT_CPU_SAFE(pVCpu);
8895
8896 /* Preparatory work for running guest code, this may force us to return
8897 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8898 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8899 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
8900 if (rcStrict != VINF_SUCCESS)
8901 break;
8902
8903 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
8904 int rcRun = hmR0VmxRunGuest(pVCpu);
8905
8906 /* Restore any residual host-state and save any bits shared between host
8907 and guest into the guest-CPU state. Re-enables interrupts! */
8908 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8909
8910 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8911 if (RT_SUCCESS(rcRun))
8912 { /* very likely */ }
8913 else
8914 {
8915 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
8916 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
8917 return rcRun;
8918 }
8919
8920 /* Profile the VM-exit. */
8921 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8922 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8923 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8924 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
8925 HMVMX_START_EXIT_DISPATCH_PROF();
8926
8927 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
8928
8929 /* Handle the VM-exit. */
8930#ifdef HMVMX_USE_FUNCTION_TABLE
8931 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
8932#else
8933 rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient, VmxTransient.uExitReason);
8934#endif
8935 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
8936 if (rcStrict == VINF_SUCCESS)
8937 {
8938 if (cLoops <= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
8939 continue; /* likely */
8940 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8941 rcStrict = VINF_EM_RAW_INTERRUPT;
8942 }
8943 break;
8944 }
8945
8946 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8947 return rcStrict;
8948}
8949
8950
8951
8952/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8953 * probes.
8954 *
8955 * The following few functions and associated structure contains the bloat
8956 * necessary for providing detailed debug events and dtrace probes as well as
8957 * reliable host side single stepping. This works on the principle of
8958 * "subclassing" the normal execution loop and workers. We replace the loop
8959 * method completely and override selected helpers to add necessary adjustments
8960 * to their core operation.
8961 *
8962 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8963 * any performance for debug and analysis features.
8964 *
8965 * @{
8966 */
8967
8968/**
8969 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
8970 * the debug run loop.
8971 */
8972typedef struct VMXRUNDBGSTATE
8973{
8974 /** The RIP we started executing at. This is for detecting that we stepped. */
8975 uint64_t uRipStart;
8976 /** The CS we started executing with. */
8977 uint16_t uCsStart;
8978
8979 /** Whether we've actually modified the 1st execution control field. */
8980 bool fModifiedProcCtls : 1;
8981 /** Whether we've actually modified the 2nd execution control field. */
8982 bool fModifiedProcCtls2 : 1;
8983 /** Whether we've actually modified the exception bitmap. */
8984 bool fModifiedXcptBitmap : 1;
8985
8986 /** We desire the modified the CR0 mask to be cleared. */
8987 bool fClearCr0Mask : 1;
8988 /** We desire the modified the CR4 mask to be cleared. */
8989 bool fClearCr4Mask : 1;
8990 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8991 uint32_t fCpe1Extra;
8992 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
8993 uint32_t fCpe1Unwanted;
8994 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
8995 uint32_t fCpe2Extra;
8996 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
8997 uint32_t bmXcptExtra;
8998 /** The sequence number of the Dtrace provider settings the state was
8999 * configured against. */
9000 uint32_t uDtraceSettingsSeqNo;
9001 /** VM-exits to check (one bit per VM-exit). */
9002 uint32_t bmExitsToCheck[3];
9003
9004 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9005 uint32_t fProcCtlsInitial;
9006 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9007 uint32_t fProcCtls2Initial;
9008 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9009 uint32_t bmXcptInitial;
9010} VMXRUNDBGSTATE;
9011AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9012typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9013
9014
9015/**
9016 * Initializes the VMXRUNDBGSTATE structure.
9017 *
9018 * @param pVCpu The cross context virtual CPU structure of the
9019 * calling EMT.
9020 * @param pDbgState The structure to initialize.
9021 */
9022static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9023{
9024 pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
9025 pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
9026
9027 pDbgState->fModifiedProcCtls = false;
9028 pDbgState->fModifiedProcCtls2 = false;
9029 pDbgState->fModifiedXcptBitmap = false;
9030 pDbgState->fClearCr0Mask = false;
9031 pDbgState->fClearCr4Mask = false;
9032 pDbgState->fCpe1Extra = 0;
9033 pDbgState->fCpe1Unwanted = 0;
9034 pDbgState->fCpe2Extra = 0;
9035 pDbgState->bmXcptExtra = 0;
9036 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9037 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9038 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9039}
9040
9041
9042/**
9043 * Updates the VMSC fields with changes requested by @a pDbgState.
9044 *
9045 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9046 * immediately before executing guest code, i.e. when interrupts are disabled.
9047 * We don't check status codes here as we cannot easily assert or return in the
9048 * latter case.
9049 *
9050 * @param pVCpu The cross context virtual CPU structure.
9051 * @param pDbgState The debug state.
9052 */
9053static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9054{
9055 /*
9056 * Ensure desired flags in VMCS control fields are set.
9057 * (Ignoring write failure here, as we're committed and it's just debug extras.)
9058 *
9059 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9060 * there should be no stale data in pCtx at this point.
9061 */
9062 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9063 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9064 {
9065 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
9066 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9067 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9068 Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9069 pDbgState->fModifiedProcCtls = true;
9070 }
9071
9072 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9073 {
9074 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9075 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9076 Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9077 pDbgState->fModifiedProcCtls2 = true;
9078 }
9079
9080 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9081 {
9082 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9083 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9084 Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9085 pDbgState->fModifiedXcptBitmap = true;
9086 }
9087
9088 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32Cr0Mask != 0)
9089 {
9090 pVCpu->hm.s.vmx.u32Cr0Mask = 0;
9091 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9092 Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9093 }
9094
9095 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32Cr4Mask != 0)
9096 {
9097 pVCpu->hm.s.vmx.u32Cr4Mask = 0;
9098 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9099 Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9100 }
9101}
9102
9103
9104/**
9105 * Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
9106 * re-entry next time around.
9107 *
9108 * @returns Strict VBox status code (i.e. informational status codes too).
9109 * @param pVCpu The cross context virtual CPU structure.
9110 * @param pDbgState The debug state.
9111 * @param rcStrict The return code from executing the guest using single
9112 * stepping.
9113 */
9114static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9115{
9116 /*
9117 * Restore VM-exit control settings as we may not reenter this function the
9118 * next time around.
9119 */
9120 /* We reload the initial value, trigger what we can of recalculations the
9121 next time around. From the looks of things, that's all that's required atm. */
9122 if (pDbgState->fModifiedProcCtls)
9123 {
9124 if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9125 pDbgState->fProcCtlsInitial |= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9126 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9127 AssertRCReturn(rc2, rc2);
9128 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9129 }
9130
9131 /* We're currently the only ones messing with this one, so just restore the
9132 cached value and reload the field. */
9133 if ( pDbgState->fModifiedProcCtls2
9134 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9135 {
9136 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9137 AssertRCReturn(rc2, rc2);
9138 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9139 }
9140
9141 /* If we've modified the exception bitmap, we restore it and trigger
9142 reloading and partial recalculation the next time around. */
9143 if (pDbgState->fModifiedXcptBitmap)
9144 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9145
9146 return rcStrict;
9147}
9148
9149
9150/**
9151 * Configures VM-exit controls for current DBGF and DTrace settings.
9152 *
9153 * This updates @a pDbgState and the VMCS execution control fields to reflect
9154 * the necessary VM-exits demanded by DBGF and DTrace.
9155 *
9156 * @param pVCpu The cross context virtual CPU structure.
9157 * @param pDbgState The debug state.
9158 * @param pVmxTransient Pointer to the VMX transient structure. May update
9159 * fUpdateTscOffsettingAndPreemptTimer.
9160 */
9161static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9162{
9163 /*
9164 * Take down the dtrace serial number so we can spot changes.
9165 */
9166 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9167 ASMCompilerBarrier();
9168
9169 /*
9170 * We'll rebuild most of the middle block of data members (holding the
9171 * current settings) as we go along here, so start by clearing it all.
9172 */
9173 pDbgState->bmXcptExtra = 0;
9174 pDbgState->fCpe1Extra = 0;
9175 pDbgState->fCpe1Unwanted = 0;
9176 pDbgState->fCpe2Extra = 0;
9177 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9178 pDbgState->bmExitsToCheck[i] = 0;
9179
9180 /*
9181 * Software interrupts (INT XXh) - no idea how to trigger these...
9182 */
9183 PVM pVM = pVCpu->CTX_SUFF(pVM);
9184 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9185 || VBOXVMM_INT_SOFTWARE_ENABLED())
9186 {
9187 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9188 }
9189
9190 /*
9191 * INT3 breakpoints - triggered by #BP exceptions.
9192 */
9193 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9194 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9195
9196 /*
9197 * Exception bitmap and XCPT events+probes.
9198 */
9199 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9200 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9201 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9202
9203 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9204 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9205 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9206 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9207 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9208 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9209 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9210 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9211 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9212 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9213 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9214 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9215 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9216 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9217 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9218 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9219 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9220 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9221
9222 if (pDbgState->bmXcptExtra)
9223 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9224
9225 /*
9226 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9227 *
9228 * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
9229 * So, when adding/changing/removing please don't forget to update it.
9230 *
9231 * Some of the macros are picking up local variables to save horizontal space,
9232 * (being able to see it in a table is the lesser evil here).
9233 */
9234#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9235 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9236 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9237#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9238 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9239 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9240 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9241 } else do { } while (0)
9242#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9243 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9244 { \
9245 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9246 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9247 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9248 } else do { } while (0)
9249#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9250 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9251 { \
9252 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9253 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9254 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9255 } else do { } while (0)
9256#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9257 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9258 { \
9259 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9260 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9261 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9262 } else do { } while (0)
9263
9264 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9265 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9266 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9267 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9268 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9269
9270 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9271 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9272 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9273 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9274 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
9275 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9276 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9277 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9278 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
9279 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9280 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
9281 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9282 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
9283 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9284 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9285 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9286 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9287 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9288 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9289 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9290 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9291 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9292 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9293 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9294 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9295 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9296 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9297 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9298 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9299 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9300 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9301 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9302 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9303 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9304 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9305 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9306
9307 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9308 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9309 {
9310 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_APIC_TPR);
9311 AssertRC(rc);
9312
9313#if 0 /** @todo fix me */
9314 pDbgState->fClearCr0Mask = true;
9315 pDbgState->fClearCr4Mask = true;
9316#endif
9317 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9318 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_STORE_EXIT | VMX_PROC_CTLS_CR8_STORE_EXIT;
9319 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9320 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_LOAD_EXIT | VMX_PROC_CTLS_CR8_LOAD_EXIT;
9321 pDbgState->fCpe1Unwanted |= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
9322 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9323 require clearing here and in the loop if we start using it. */
9324 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9325 }
9326 else
9327 {
9328 if (pDbgState->fClearCr0Mask)
9329 {
9330 pDbgState->fClearCr0Mask = false;
9331 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9332 }
9333 if (pDbgState->fClearCr4Mask)
9334 {
9335 pDbgState->fClearCr4Mask = false;
9336 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9337 }
9338 }
9339 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9340 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9341
9342 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9343 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9344 {
9345 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9346 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9347 }
9348 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9349 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9350
9351 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
9352 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9353 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
9354 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9355 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
9356 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9357 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
9358 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9359#if 0 /** @todo too slow, fix handler. */
9360 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
9361#endif
9362 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9363
9364 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9365 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9366 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9367 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9368 {
9369 pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
9370 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
9371 }
9372 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9373 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9374 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9375 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9376
9377 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9378 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9379 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9380 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9381 {
9382 pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
9383 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
9384 }
9385 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
9386 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
9387 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
9388 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
9389
9390 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9391 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9392 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
9393 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9394 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9395 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9396 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
9397 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9398 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9399 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9400 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
9401 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9402 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
9403 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9404 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9405 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9406 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
9407 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9408 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9409 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9410 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9411 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9412
9413#undef IS_EITHER_ENABLED
9414#undef SET_ONLY_XBM_IF_EITHER_EN
9415#undef SET_CPE1_XBM_IF_EITHER_EN
9416#undef SET_CPEU_XBM_IF_EITHER_EN
9417#undef SET_CPE2_XBM_IF_EITHER_EN
9418
9419 /*
9420 * Sanitize the control stuff.
9421 */
9422 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
9423 if (pDbgState->fCpe2Extra)
9424 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
9425 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
9426 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0;
9427 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
9428 {
9429 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9430 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9431 }
9432
9433 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9434 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9435 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9436 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9437}
9438
9439
9440/**
9441 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9442 * appropriate.
9443 *
9444 * The caller has checked the VM-exit against the
9445 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9446 * already, so we don't have to do that either.
9447 *
9448 * @returns Strict VBox status code (i.e. informational status codes too).
9449 * @param pVCpu The cross context virtual CPU structure.
9450 * @param pVmxTransient Pointer to the VMX-transient structure.
9451 * @param uExitReason The VM-exit reason.
9452 *
9453 * @remarks The name of this function is displayed by dtrace, so keep it short
9454 * and to the point. No longer than 33 chars long, please.
9455 */
9456static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9457{
9458 /*
9459 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9460 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9461 *
9462 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9463 * does. Must add/change/remove both places. Same ordering, please.
9464 *
9465 * Added/removed events must also be reflected in the next section
9466 * where we dispatch dtrace events.
9467 */
9468 bool fDtrace1 = false;
9469 bool fDtrace2 = false;
9470 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9471 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9472 uint32_t uEventArg = 0;
9473#define SET_EXIT(a_EventSubName) \
9474 do { \
9475 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9476 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9477 } while (0)
9478#define SET_BOTH(a_EventSubName) \
9479 do { \
9480 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9481 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9482 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9483 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9484 } while (0)
9485 switch (uExitReason)
9486 {
9487 case VMX_EXIT_MTF:
9488 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9489
9490 case VMX_EXIT_XCPT_OR_NMI:
9491 {
9492 uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9493 switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
9494 {
9495 case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
9496 case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
9497 case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
9498 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9499 {
9500 if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
9501 {
9502 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9503 uEventArg = pVmxTransient->uExitIntErrorCode;
9504 }
9505 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9506 switch (enmEvent1)
9507 {
9508 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9509 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9510 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9511 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9512 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9513 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9514 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9515 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9516 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9517 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9518 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9519 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9520 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9521 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9522 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9523 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9524 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9525 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9526 default: break;
9527 }
9528 }
9529 else
9530 AssertFailed();
9531 break;
9532
9533 case VMX_EXIT_INT_INFO_TYPE_SW_INT:
9534 uEventArg = idxVector;
9535 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9536 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9537 break;
9538 }
9539 break;
9540 }
9541
9542 case VMX_EXIT_TRIPLE_FAULT:
9543 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9544 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9545 break;
9546 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9547 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9548 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9549 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9550 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9551
9552 /* Instruction specific VM-exits: */
9553 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9554 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9555 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9556 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9557 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9558 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9559 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9560 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9561 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9562 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9563 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9564 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9565 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9566 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9567 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9568 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9569 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9570 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9571 case VMX_EXIT_MOV_CRX:
9572 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9573 if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9574 SET_BOTH(CRX_READ);
9575 else
9576 SET_BOTH(CRX_WRITE);
9577 uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
9578 break;
9579 case VMX_EXIT_MOV_DRX:
9580 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9581 if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
9582 == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9583 SET_BOTH(DRX_READ);
9584 else
9585 SET_BOTH(DRX_WRITE);
9586 uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
9587 break;
9588 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9589 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9590 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9591 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9592 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9593 case VMX_EXIT_GDTR_IDTR_ACCESS:
9594 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9595 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
9596 {
9597 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9598 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9599 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9600 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9601 }
9602 break;
9603
9604 case VMX_EXIT_LDTR_TR_ACCESS:
9605 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9606 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
9607 {
9608 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9609 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9610 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9611 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9612 }
9613 break;
9614
9615 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9616 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9617 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9618 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9619 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9620 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9621 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9622 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9623 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9624 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9625 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9626
9627 /* Events that aren't relevant at this point. */
9628 case VMX_EXIT_EXT_INT:
9629 case VMX_EXIT_INT_WINDOW:
9630 case VMX_EXIT_NMI_WINDOW:
9631 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9632 case VMX_EXIT_PREEMPT_TIMER:
9633 case VMX_EXIT_IO_INSTR:
9634 break;
9635
9636 /* Errors and unexpected events. */
9637 case VMX_EXIT_INIT_SIGNAL:
9638 case VMX_EXIT_SIPI:
9639 case VMX_EXIT_IO_SMI:
9640 case VMX_EXIT_SMI:
9641 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9642 case VMX_EXIT_ERR_MSR_LOAD:
9643 case VMX_EXIT_ERR_MACHINE_CHECK:
9644 break;
9645
9646 default:
9647 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9648 break;
9649 }
9650#undef SET_BOTH
9651#undef SET_EXIT
9652
9653 /*
9654 * Dtrace tracepoints go first. We do them here at once so we don't
9655 * have to copy the guest state saving and stuff a few dozen times.
9656 * Down side is that we've got to repeat the switch, though this time
9657 * we use enmEvent since the probes are a subset of what DBGF does.
9658 */
9659 if (fDtrace1 || fDtrace2)
9660 {
9661 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9662 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9663 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9664 switch (enmEvent1)
9665 {
9666 /** @todo consider which extra parameters would be helpful for each probe. */
9667 case DBGFEVENT_END: break;
9668 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
9669 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
9670 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
9671 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
9672 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
9673 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
9674 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
9675 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
9676 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
9677 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
9678 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
9679 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
9680 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
9681 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
9682 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
9683 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
9684 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
9685 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
9686 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9687 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9688 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
9689 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
9690 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
9691 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
9692 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
9693 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
9694 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
9695 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9696 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9697 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9698 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9699 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9700 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
9701 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9702 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
9703 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
9704 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
9705 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
9706 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
9707 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
9708 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
9709 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
9710 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
9711 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
9712 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
9713 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
9714 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
9715 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
9716 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
9717 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
9718 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
9719 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
9720 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
9721 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
9722 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
9723 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
9724 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
9725 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
9726 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
9727 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
9728 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
9729 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
9730 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
9731 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
9732 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
9733 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
9734 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9735 }
9736 switch (enmEvent2)
9737 {
9738 /** @todo consider which extra parameters would be helpful for each probe. */
9739 case DBGFEVENT_END: break;
9740 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
9741 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9742 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
9743 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
9744 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
9745 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
9746 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
9747 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
9748 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
9749 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9750 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9751 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9752 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9753 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9754 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
9755 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9756 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
9757 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
9758 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
9759 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
9760 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
9761 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
9762 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
9763 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
9764 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
9765 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
9766 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
9767 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
9768 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
9769 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
9770 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
9771 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
9772 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
9773 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
9774 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
9775 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
9776 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
9777 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
9778 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
9779 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
9780 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
9781 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
9782 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
9783 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
9784 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
9785 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
9786 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
9787 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
9788 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
9789 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
9790 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
9791 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
9792 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9793 }
9794 }
9795
9796 /*
9797 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9798 * the DBGF call will do a full check).
9799 *
9800 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9801 * Note! If we have to events, we prioritize the first, i.e. the instruction
9802 * one, in order to avoid event nesting.
9803 */
9804 PVM pVM = pVCpu->CTX_SUFF(pVM);
9805 if ( enmEvent1 != DBGFEVENT_END
9806 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9807 {
9808 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
9809 if (rcStrict != VINF_SUCCESS)
9810 return rcStrict;
9811 }
9812 else if ( enmEvent2 != DBGFEVENT_END
9813 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9814 {
9815 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
9816 if (rcStrict != VINF_SUCCESS)
9817 return rcStrict;
9818 }
9819
9820 return VINF_SUCCESS;
9821}
9822
9823
9824/**
9825 * Single-stepping VM-exit filtering.
9826 *
9827 * This is preprocessing the VM-exits and deciding whether we've gotten far
9828 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9829 * handling is performed.
9830 *
9831 * @returns Strict VBox status code (i.e. informational status codes too).
9832 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9833 * @param pVmxTransient Pointer to the VMX-transient structure.
9834 * @param pDbgState The debug state.
9835 */
9836DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
9837{
9838 /*
9839 * Expensive (saves context) generic dtrace VM-exit probe.
9840 */
9841 uint32_t const uExitReason = pVmxTransient->uExitReason;
9842 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9843 { /* more likely */ }
9844 else
9845 {
9846 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9847 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9848 AssertRC(rc);
9849 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
9850 }
9851
9852 /*
9853 * Check for host NMI, just to get that out of the way.
9854 */
9855 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9856 { /* normally likely */ }
9857 else
9858 {
9859 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9860 AssertRCReturn(rc2, rc2);
9861 uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
9862 if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
9863 return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
9864 }
9865
9866 /*
9867 * Check for single stepping event if we're stepping.
9868 */
9869 if (pVCpu->hm.s.fSingleInstruction)
9870 {
9871 switch (uExitReason)
9872 {
9873 case VMX_EXIT_MTF:
9874 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9875
9876 /* Various events: */
9877 case VMX_EXIT_XCPT_OR_NMI:
9878 case VMX_EXIT_EXT_INT:
9879 case VMX_EXIT_TRIPLE_FAULT:
9880 case VMX_EXIT_INT_WINDOW:
9881 case VMX_EXIT_NMI_WINDOW:
9882 case VMX_EXIT_TASK_SWITCH:
9883 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9884 case VMX_EXIT_APIC_ACCESS:
9885 case VMX_EXIT_EPT_VIOLATION:
9886 case VMX_EXIT_EPT_MISCONFIG:
9887 case VMX_EXIT_PREEMPT_TIMER:
9888
9889 /* Instruction specific VM-exits: */
9890 case VMX_EXIT_CPUID:
9891 case VMX_EXIT_GETSEC:
9892 case VMX_EXIT_HLT:
9893 case VMX_EXIT_INVD:
9894 case VMX_EXIT_INVLPG:
9895 case VMX_EXIT_RDPMC:
9896 case VMX_EXIT_RDTSC:
9897 case VMX_EXIT_RSM:
9898 case VMX_EXIT_VMCALL:
9899 case VMX_EXIT_VMCLEAR:
9900 case VMX_EXIT_VMLAUNCH:
9901 case VMX_EXIT_VMPTRLD:
9902 case VMX_EXIT_VMPTRST:
9903 case VMX_EXIT_VMREAD:
9904 case VMX_EXIT_VMRESUME:
9905 case VMX_EXIT_VMWRITE:
9906 case VMX_EXIT_VMXOFF:
9907 case VMX_EXIT_VMXON:
9908 case VMX_EXIT_MOV_CRX:
9909 case VMX_EXIT_MOV_DRX:
9910 case VMX_EXIT_IO_INSTR:
9911 case VMX_EXIT_RDMSR:
9912 case VMX_EXIT_WRMSR:
9913 case VMX_EXIT_MWAIT:
9914 case VMX_EXIT_MONITOR:
9915 case VMX_EXIT_PAUSE:
9916 case VMX_EXIT_GDTR_IDTR_ACCESS:
9917 case VMX_EXIT_LDTR_TR_ACCESS:
9918 case VMX_EXIT_INVEPT:
9919 case VMX_EXIT_RDTSCP:
9920 case VMX_EXIT_INVVPID:
9921 case VMX_EXIT_WBINVD:
9922 case VMX_EXIT_XSETBV:
9923 case VMX_EXIT_RDRAND:
9924 case VMX_EXIT_INVPCID:
9925 case VMX_EXIT_VMFUNC:
9926 case VMX_EXIT_RDSEED:
9927 case VMX_EXIT_XSAVES:
9928 case VMX_EXIT_XRSTORS:
9929 {
9930 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
9931 AssertRCReturn(rc, rc);
9932 if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
9933 || pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
9934 return VINF_EM_DBG_STEPPED;
9935 break;
9936 }
9937
9938 /* Errors and unexpected events: */
9939 case VMX_EXIT_INIT_SIGNAL:
9940 case VMX_EXIT_SIPI:
9941 case VMX_EXIT_IO_SMI:
9942 case VMX_EXIT_SMI:
9943 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9944 case VMX_EXIT_ERR_MSR_LOAD:
9945 case VMX_EXIT_ERR_MACHINE_CHECK:
9946 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9947 break;
9948
9949 default:
9950 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9951 break;
9952 }
9953 }
9954
9955 /*
9956 * Check for debugger event breakpoints and dtrace probes.
9957 */
9958 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9959 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9960 {
9961 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
9962 if (rcStrict != VINF_SUCCESS)
9963 return rcStrict;
9964 }
9965
9966 /*
9967 * Normal processing.
9968 */
9969#ifdef HMVMX_USE_FUNCTION_TABLE
9970 return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
9971#else
9972 return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
9973#endif
9974}
9975
9976
9977/**
9978 * Single steps guest code using VT-x.
9979 *
9980 * @returns Strict VBox status code (i.e. informational status codes too).
9981 * @param pVCpu The cross context virtual CPU structure.
9982 *
9983 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
9984 */
9985static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu)
9986{
9987 VMXTRANSIENT VmxTransient;
9988 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9989
9990 /* Set HMCPU indicators. */
9991 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
9992 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
9993 pVCpu->hm.s.fDebugWantRdTscExit = false;
9994 pVCpu->hm.s.fUsingDebugLoop = true;
9995
9996 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
9997 VMXRUNDBGSTATE DbgState;
9998 hmR0VmxRunDebugStateInit(pVCpu, &DbgState);
9999 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
10000
10001 /*
10002 * The loop.
10003 */
10004 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10005 for (uint32_t cLoops = 0; ; cLoops++)
10006 {
10007 Assert(!HMR0SuspendPending());
10008 HMVMX_ASSERT_CPU_SAFE(pVCpu);
10009 bool fStepping = pVCpu->hm.s.fSingleInstruction;
10010
10011 /*
10012 * Preparatory work for running guest code, this may force us to return
10013 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10014 */
10015 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10016 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10017 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
10018 if (rcStrict != VINF_SUCCESS)
10019 break;
10020
10021 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10022 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10023
10024 /*
10025 * Now we can run the guest code.
10026 */
10027 int rcRun = hmR0VmxRunGuest(pVCpu);
10028
10029 /*
10030 * Restore any residual host-state and save any bits shared between host
10031 * and guest into the guest-CPU state. Re-enables interrupts!
10032 */
10033 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10034
10035 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10036 if (RT_SUCCESS(rcRun))
10037 { /* very likely */ }
10038 else
10039 {
10040 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10041 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10042 return rcRun;
10043 }
10044
10045 /* Profile the VM-exit. */
10046 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10047 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10048 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10049 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10050 HMVMX_START_EXIT_DISPATCH_PROF();
10051
10052 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10053
10054 /*
10055 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10056 */
10057 rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
10058 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10059 if (rcStrict != VINF_SUCCESS)
10060 break;
10061 if (cLoops > pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
10062 {
10063 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10064 rcStrict = VINF_EM_RAW_INTERRUPT;
10065 break;
10066 }
10067
10068 /*
10069 * Stepping: Did the RIP change, if so, consider it a single step.
10070 * Otherwise, make sure one of the TFs gets set.
10071 */
10072 if (fStepping)
10073 {
10074 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
10075 AssertRC(rc);
10076 if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
10077 || pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
10078 {
10079 rcStrict = VINF_EM_DBG_STEPPED;
10080 break;
10081 }
10082 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
10083 }
10084
10085 /*
10086 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10087 */
10088 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10089 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
10090 }
10091
10092 /*
10093 * Clear the X86_EFL_TF if necessary.
10094 */
10095 if (pVCpu->hm.s.fClearTrapFlag)
10096 {
10097 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
10098 AssertRC(rc);
10099 pVCpu->hm.s.fClearTrapFlag = false;
10100 pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
10101 }
10102 /** @todo there seems to be issues with the resume flag when the monitor trap
10103 * flag is pending without being used. Seen early in bios init when
10104 * accessing APIC page in protected mode. */
10105
10106 /*
10107 * Restore VM-exit control settings as we may not reenter this function the
10108 * next time around.
10109 */
10110 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10111
10112 /* Restore HMCPU indicators. */
10113 pVCpu->hm.s.fUsingDebugLoop = false;
10114 pVCpu->hm.s.fDebugWantRdTscExit = false;
10115 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10116
10117 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10118 return rcStrict;
10119}
10120
10121
10122/** @} */
10123
10124
10125/**
10126 * Checks if any expensive dtrace probes are enabled and we should go to the
10127 * debug loop.
10128 *
10129 * @returns true if we should use debug loop, false if not.
10130 */
10131static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10132{
10133 /* It's probably faster to OR the raw 32-bit counter variables together.
10134 Since the variables are in an array and the probes are next to one
10135 another (more or less), we have good locality. So, better read
10136 eight-nine cache lines ever time and only have one conditional, than
10137 128+ conditionals, right? */
10138 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10139 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10140 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10141 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10142 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10143 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10144 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10145 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10146 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10147 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10148 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10149 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10150 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10151 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10152 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10153 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10154 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10155 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10156 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10157 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10158 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10159 ) != 0
10160 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10161 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10162 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10163 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10164 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10165 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10166 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10167 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10168 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10169 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10170 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10171 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10172 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10173 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10174 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10175 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10176 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10177 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10178 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10179 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10180 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10181 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10182 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10183 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10184 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10185 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10186 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10187 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10188 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10189 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10190 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10191 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10192 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10193 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10194 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10195 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10196 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10197 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10198 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10199 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10200 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10201 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10202 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10203 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10204 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10205 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10206 ) != 0
10207 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10208 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10209 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10210 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10211 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10212 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10213 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10214 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10215 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10216 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10217 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10218 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10219 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10220 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10221 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10222 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10223 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10224 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10225 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10226 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10227 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10228 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10229 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10230 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10231 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10232 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10233 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10234 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10235 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10236 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10237 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10238 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10239 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10240 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10241 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10242 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10243 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10244 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10245 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10246 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10247 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10248 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10249 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10250 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10251 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10252 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10253 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10254 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10255 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10256 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10257 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10258 ) != 0;
10259}
10260
10261
10262/**
10263 * Runs the guest code using VT-x.
10264 *
10265 * @returns Strict VBox status code (i.e. informational status codes too).
10266 * @param pVCpu The cross context virtual CPU structure.
10267 */
10268VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
10269{
10270 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10271 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10272 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10273 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
10274
10275 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10276
10277 VBOXSTRICTRC rcStrict;
10278 if ( !pVCpu->hm.s.fUseDebugLoop
10279 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10280 && !DBGFIsStepping(pVCpu)
10281 && !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
10282 rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu);
10283 else
10284 rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu);
10285
10286 if (rcStrict == VERR_EM_INTERPRETER)
10287 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10288 else if (rcStrict == VINF_EM_RESET)
10289 rcStrict = VINF_EM_TRIPLE_FAULT;
10290
10291 int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
10292 if (RT_FAILURE(rc2))
10293 {
10294 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10295 rcStrict = rc2;
10296 }
10297 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10298 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10299 return rcStrict;
10300}
10301
10302
10303#ifndef HMVMX_USE_FUNCTION_TABLE
10304DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10305{
10306#ifdef DEBUG_ramshankar
10307#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10308 do { \
10309 if (a_fSave != 0) \
10310 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10311 VBOXSTRICTRC rcStrict = a_CallExpr; \
10312 if (a_fSave != 0) \
10313 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10314 return rcStrict; \
10315 } while (0)
10316#else
10317# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10318#endif
10319 switch (rcReason)
10320 {
10321 case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
10322 case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
10323 case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
10324 case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
10325 case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
10326 case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
10327 case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
10328 case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
10329 case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
10330 case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
10331 case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
10332 case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
10333 case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
10334 case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
10335 case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
10336 case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
10337 case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
10338 case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
10339 case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
10340 case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
10341 case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
10342 case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
10343 case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
10344 case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pVmxTransient));
10345 case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
10346 case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
10347 case VMX_EXIT_GDTR_IDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10348 case VMX_EXIT_LDTR_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10349 case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
10350 case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
10351 case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
10352 case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
10353 case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
10354 case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
10355
10356 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
10357 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
10358 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pVmxTransient);
10359 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pVmxTransient);
10360 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pVmxTransient);
10361 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pVmxTransient);
10362 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pVmxTransient);
10363 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
10364 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pVmxTransient);
10365
10366 case VMX_EXIT_VMCLEAR:
10367 case VMX_EXIT_VMLAUNCH:
10368 case VMX_EXIT_VMPTRLD:
10369 case VMX_EXIT_VMPTRST:
10370 case VMX_EXIT_VMREAD:
10371 case VMX_EXIT_VMRESUME:
10372 case VMX_EXIT_VMWRITE:
10373 case VMX_EXIT_VMXOFF:
10374 case VMX_EXIT_VMXON:
10375 case VMX_EXIT_INVEPT:
10376 case VMX_EXIT_INVVPID:
10377 case VMX_EXIT_VMFUNC:
10378 case VMX_EXIT_XSAVES:
10379 case VMX_EXIT_XRSTORS:
10380 return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
10381
10382 case VMX_EXIT_ENCLS:
10383 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10384 case VMX_EXIT_PML_FULL:
10385 default:
10386 return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
10387 }
10388#undef VMEXIT_CALL_RET
10389}
10390#endif /* !HMVMX_USE_FUNCTION_TABLE */
10391
10392
10393#ifdef VBOX_STRICT
10394/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10395# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10396 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10397
10398# define HMVMX_ASSERT_PREEMPT_CPUID() \
10399 do { \
10400 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10401 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10402 } while (0)
10403
10404# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10405 do { \
10406 AssertPtr((a_pVCpu)); \
10407 AssertPtr((a_pVmxTransient)); \
10408 Assert((a_pVmxTransient)->fVMEntryFailed == false); \
10409 Assert(ASMIntAreEnabled()); \
10410 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10411 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10412 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
10413 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10414 if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
10415 HMVMX_ASSERT_PREEMPT_CPUID(); \
10416 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10417 } while (0)
10418
10419# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10420 do { \
10421 Log4Func(("\n")); \
10422 } while (0)
10423#else
10424# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10425 do { \
10426 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10427 NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
10428 } while (0)
10429# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
10430#endif
10431
10432
10433/**
10434 * Advances the guest RIP by the specified number of bytes.
10435 *
10436 * @param pVCpu The cross context virtual CPU structure.
10437 * @param cbInstr Number of bytes to advance the RIP by.
10438 *
10439 * @remarks No-long-jump zone!!!
10440 */
10441DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
10442{
10443 /* Advance the RIP. */
10444 pVCpu->cpum.GstCtx.rip += cbInstr;
10445 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10446
10447 /* Update interrupt inhibition. */
10448 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10449 && pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
10450 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10451}
10452
10453
10454/**
10455 * Advances the guest RIP after reading it from the VMCS.
10456 *
10457 * @returns VBox status code, no informational status codes.
10458 * @param pVCpu The cross context virtual CPU structure.
10459 * @param pVmxTransient Pointer to the VMX transient structure.
10460 *
10461 * @remarks No-long-jump zone!!!
10462 */
10463static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10464{
10465 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10466 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS);
10467 AssertRCReturn(rc, rc);
10468
10469 hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
10470
10471 /*
10472 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10473 * pending debug exception field as it takes care of priority of events.
10474 *
10475 * See Intel spec. 32.2.1 "Debug Exceptions".
10476 */
10477 if ( !pVCpu->hm.s.fSingleInstruction
10478 && pVCpu->cpum.GstCtx.eflags.Bits.u1TF)
10479 {
10480 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10481 AssertRCReturn(rc, rc);
10482 }
10483
10484 return VINF_SUCCESS;
10485}
10486
10487
10488/**
10489 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10490 * and update error record fields accordingly.
10491 *
10492 * @return VMX_IGS_* return codes.
10493 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10494 * wrong with the guest state.
10495 *
10496 * @param pVCpu The cross context virtual CPU structure.
10497 *
10498 * @remarks This function assumes our cache of the VMCS controls
10499 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10500 */
10501static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu)
10502{
10503#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10504#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10505 uError = (err); \
10506 break; \
10507 } else do { } while (0)
10508
10509 int rc;
10510 PVM pVM = pVCpu->CTX_SUFF(pVM);
10511 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10512 uint32_t uError = VMX_IGS_ERROR;
10513 uint32_t u32Val;
10514 bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10515
10516 do
10517 {
10518 /*
10519 * CR0.
10520 */
10521 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10522 uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10523 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10524 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10525 if (fUnrestrictedGuest)
10526 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
10527
10528 uint32_t u32GuestCr0;
10529 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10530 AssertRCBreak(rc);
10531 HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10532 HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10533 if ( !fUnrestrictedGuest
10534 && (u32GuestCr0 & X86_CR0_PG)
10535 && !(u32GuestCr0 & X86_CR0_PE))
10536 {
10537 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10538 }
10539
10540 /*
10541 * CR4.
10542 */
10543 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10544 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10545
10546 uint32_t u32GuestCr4;
10547 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10548 AssertRCBreak(rc);
10549 HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10550 HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10551
10552 /*
10553 * IA32_DEBUGCTL MSR.
10554 */
10555 uint64_t u64Val;
10556 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10557 AssertRCBreak(rc);
10558 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10559 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10560 {
10561 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10562 }
10563 uint64_t u64DebugCtlMsr = u64Val;
10564
10565#ifdef VBOX_STRICT
10566 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10567 AssertRCBreak(rc);
10568 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10569#endif
10570 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
10571
10572 /*
10573 * RIP and RFLAGS.
10574 */
10575 uint32_t u32Eflags;
10576#if HC_ARCH_BITS == 64
10577 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10578 AssertRCBreak(rc);
10579 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10580 if ( !fLongModeGuest
10581 || !pCtx->cs.Attr.n.u1Long)
10582 {
10583 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10584 }
10585 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10586 * must be identical if the "IA-32e mode guest" VM-entry
10587 * control is 1 and CS.L is 1. No check applies if the
10588 * CPU supports 64 linear-address bits. */
10589
10590 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10591 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10592 AssertRCBreak(rc);
10593 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10594 VMX_IGS_RFLAGS_RESERVED);
10595 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10596 u32Eflags = u64Val;
10597#else
10598 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10599 AssertRCBreak(rc);
10600 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10601 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10602#endif
10603
10604 if ( fLongModeGuest
10605 || ( fUnrestrictedGuest
10606 && !(u32GuestCr0 & X86_CR0_PE)))
10607 {
10608 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10609 }
10610
10611 uint32_t u32EntryInfo;
10612 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10613 AssertRCBreak(rc);
10614 if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
10615 && VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
10616 {
10617 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10618 }
10619
10620 /*
10621 * 64-bit checks.
10622 */
10623#if HC_ARCH_BITS == 64
10624 if (fLongModeGuest)
10625 {
10626 HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10627 HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10628 }
10629
10630 if ( !fLongModeGuest
10631 && (u32GuestCr4 & X86_CR4_PCIDE))
10632 {
10633 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10634 }
10635
10636 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10637 * 51:32 beyond the processor's physical-address width are 0. */
10638
10639 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10640 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10641 {
10642 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10643 }
10644
10645 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10646 AssertRCBreak(rc);
10647 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10648
10649 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10650 AssertRCBreak(rc);
10651 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10652#endif
10653
10654 /*
10655 * PERF_GLOBAL MSR.
10656 */
10657 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
10658 {
10659 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10660 AssertRCBreak(rc);
10661 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10662 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10663 }
10664
10665 /*
10666 * PAT MSR.
10667 */
10668 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
10669 {
10670 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10671 AssertRCBreak(rc);
10672 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10673 for (unsigned i = 0; i < 8; i++)
10674 {
10675 uint8_t u8Val = (u64Val & 0xff);
10676 if ( u8Val != 0 /* UC */
10677 && u8Val != 1 /* WC */
10678 && u8Val != 4 /* WT */
10679 && u8Val != 5 /* WP */
10680 && u8Val != 6 /* WB */
10681 && u8Val != 7 /* UC- */)
10682 {
10683 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10684 }
10685 u64Val >>= 8;
10686 }
10687 }
10688
10689 /*
10690 * EFER MSR.
10691 */
10692 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
10693 {
10694 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10695 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10696 AssertRCBreak(rc);
10697 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10698 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10699 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10700 & VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
10701 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10702 /** @todo r=ramshankar: Unrestricted check here is probably wrong, see
10703 * iemVmxVmentryCheckGuestState(). */
10704 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10705 || !(u32GuestCr0 & X86_CR0_PG)
10706 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10707 VMX_IGS_EFER_LMA_LME_MISMATCH);
10708 }
10709
10710 /*
10711 * Segment registers.
10712 */
10713 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10714 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10715 if (!(u32Eflags & X86_EFL_VM))
10716 {
10717 /* CS */
10718 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10719 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10720 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10721 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10722 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10723 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10724 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10725 /* CS cannot be loaded with NULL in protected mode. */
10726 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10727 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10728 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10729 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10730 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10731 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10732 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10733 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10734 else
10735 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10736
10737 /* SS */
10738 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10739 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10740 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10741 if ( !(pCtx->cr0 & X86_CR0_PE)
10742 || pCtx->cs.Attr.n.u4Type == 3)
10743 {
10744 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10745 }
10746 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10747 {
10748 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10749 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10750 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10751 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10752 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10753 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10754 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10755 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10756 }
10757
10758 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmenReg(). */
10759 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10760 {
10761 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10762 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10763 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10764 || pCtx->ds.Attr.n.u4Type > 11
10765 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10766 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10767 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10768 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10769 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10770 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10771 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10772 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10773 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10774 }
10775 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10776 {
10777 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10778 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10779 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10780 || pCtx->es.Attr.n.u4Type > 11
10781 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10782 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10783 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10784 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10785 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10786 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10787 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10788 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10789 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10790 }
10791 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10792 {
10793 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10794 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10795 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10796 || pCtx->fs.Attr.n.u4Type > 11
10797 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10798 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10799 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10800 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10801 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10802 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10803 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10804 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10805 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10806 }
10807 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10808 {
10809 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10810 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10811 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10812 || pCtx->gs.Attr.n.u4Type > 11
10813 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10814 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10815 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10816 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10817 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10818 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10819 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10820 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10821 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10822 }
10823 /* 64-bit capable CPUs. */
10824#if HC_ARCH_BITS == 64
10825 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10826 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10827 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10828 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10829 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10830 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10831 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10832 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10833 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10834 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10835 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10836#endif
10837 }
10838 else
10839 {
10840 /* V86 mode checks. */
10841 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10842 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10843 {
10844 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10845 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10846 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10847 }
10848 else
10849 {
10850 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10851 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10852 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10853 }
10854
10855 /* CS */
10856 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10857 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10858 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10859 /* SS */
10860 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10861 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10862 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10863 /* DS */
10864 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10865 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10866 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10867 /* ES */
10868 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10869 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10870 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10871 /* FS */
10872 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10873 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10874 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10875 /* GS */
10876 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10877 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10878 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10879 /* 64-bit capable CPUs. */
10880#if HC_ARCH_BITS == 64
10881 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10882 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10883 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10884 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10885 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10886 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10887 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10888 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10889 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10890 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10891 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10892#endif
10893 }
10894
10895 /*
10896 * TR.
10897 */
10898 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10899 /* 64-bit capable CPUs. */
10900#if HC_ARCH_BITS == 64
10901 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10902#endif
10903 if (fLongModeGuest)
10904 {
10905 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10906 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10907 }
10908 else
10909 {
10910 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10911 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10912 VMX_IGS_TR_ATTR_TYPE_INVALID);
10913 }
10914 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10915 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10916 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10917 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10918 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10919 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10920 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10921 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10922
10923 /*
10924 * GDTR and IDTR.
10925 */
10926#if HC_ARCH_BITS == 64
10927 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10928 AssertRCBreak(rc);
10929 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10930
10931 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10932 AssertRCBreak(rc);
10933 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10934#endif
10935
10936 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10937 AssertRCBreak(rc);
10938 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10939
10940 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10941 AssertRCBreak(rc);
10942 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10943
10944 /*
10945 * Guest Non-Register State.
10946 */
10947 /* Activity State. */
10948 uint32_t u32ActivityState;
10949 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10950 AssertRCBreak(rc);
10951 HMVMX_CHECK_BREAK( !u32ActivityState
10952 || (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
10953 VMX_IGS_ACTIVITY_STATE_INVALID);
10954 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10955 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10956 uint32_t u32IntrState;
10957 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
10958 AssertRCBreak(rc);
10959 if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
10960 || u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10961 {
10962 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10963 }
10964
10965 /** @todo Activity state and injecting interrupts. Left as a todo since we
10966 * currently don't use activity states but ACTIVE. */
10967
10968 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10969 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10970
10971 /* Guest interruptibility-state. */
10972 HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10973 HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
10974 != (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10975 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10976 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10977 || !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10978 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10979 if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
10980 {
10981 if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
10982 {
10983 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10984 && !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10985 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10986 }
10987 else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
10988 {
10989 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10990 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10991 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10992 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10993 }
10994 }
10995 /** @todo Assumes the processor is not in SMM. */
10996 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10997 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10998 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10999 || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
11000 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11001 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
11002 && VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
11003 && VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
11004 {
11005 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
11006 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11007 }
11008
11009 /* Pending debug exceptions. */
11010#if HC_ARCH_BITS == 64
11011 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
11012 AssertRCBreak(rc);
11013 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11014 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11015 u32Val = u64Val; /* For pending debug exceptions checks below. */
11016#else
11017 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
11018 AssertRCBreak(rc);
11019 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11020 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11021#endif
11022
11023 if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
11024 || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
11025 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11026 {
11027 if ( (u32Eflags & X86_EFL_TF)
11028 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11029 {
11030 /* Bit 14 is PendingDebug.BS. */
11031 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11032 }
11033 if ( !(u32Eflags & X86_EFL_TF)
11034 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11035 {
11036 /* Bit 14 is PendingDebug.BS. */
11037 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11038 }
11039 }
11040
11041 /* VMCS link pointer. */
11042 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11043 AssertRCBreak(rc);
11044 if (u64Val != UINT64_C(0xffffffffffffffff))
11045 {
11046 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11047 /** @todo Bits beyond the processor's physical-address width MBZ. */
11048 /** @todo 32-bit located in memory referenced by value of this field (as a
11049 * physical address) must contain the processor's VMCS revision ID. */
11050 /** @todo SMM checks. */
11051 }
11052
11053 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11054 * not using Nested Paging? */
11055 if ( pVM->hm.s.fNestedPaging
11056 && !fLongModeGuest
11057 && CPUMIsGuestInPAEModeEx(pCtx))
11058 {
11059 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11060 AssertRCBreak(rc);
11061 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11062
11063 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11064 AssertRCBreak(rc);
11065 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11066
11067 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11068 AssertRCBreak(rc);
11069 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11070
11071 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11072 AssertRCBreak(rc);
11073 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11074 }
11075
11076 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11077 if (uError == VMX_IGS_ERROR)
11078 uError = VMX_IGS_REASON_NOT_FOUND;
11079 } while (0);
11080
11081 pVCpu->hm.s.u32HMError = uError;
11082 return uError;
11083
11084#undef HMVMX_ERROR_BREAK
11085#undef HMVMX_CHECK_BREAK
11086}
11087
11088
11089/** @name VM-exit handlers.
11090 * @{
11091 */
11092/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11093/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11094/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11095
11096/**
11097 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11098 */
11099HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11100{
11101 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11102 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11103 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11104 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11105 return VINF_SUCCESS;
11106 return VINF_EM_RAW_INTERRUPT;
11107}
11108
11109
11110/**
11111 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11112 */
11113HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11114{
11115 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11116 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11117
11118 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11119 AssertRCReturn(rc, rc);
11120
11121 uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
11122 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
11123 && uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
11124 Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11125
11126 if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
11127 {
11128 /*
11129 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
11130 * injected it ourselves and anything we inject is not going to cause a VM-exit directly
11131 * for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
11132 *
11133 * [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11134 * [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
11135 */
11136 VMXDispatchHostNmi();
11137 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11138 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11139 return VINF_SUCCESS;
11140 }
11141
11142 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11143 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
11144 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11145 { /* likely */ }
11146 else
11147 {
11148 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11149 rcStrictRc1 = VINF_SUCCESS;
11150 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11151 return rcStrictRc1;
11152 }
11153
11154 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11155 uint32_t uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
11156 switch (uIntType)
11157 {
11158 case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11159 Assert(uVector == X86_XCPT_DB);
11160 RT_FALL_THRU();
11161 case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11162 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
11163 RT_FALL_THRU();
11164 case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11165 {
11166 /*
11167 * If there's any exception caused as a result of event injection, the resulting
11168 * secondary/final execption will be pending, we shall continue guest execution
11169 * after injecting the event. The page-fault case is complicated and we manually
11170 * handle any currently pending event in hmR0VmxExitXcptPF.
11171 */
11172 if (!pVCpu->hm.s.Event.fPending)
11173 { /* likely */ }
11174 else if (uVector != X86_XCPT_PF)
11175 {
11176 rc = VINF_SUCCESS;
11177 break;
11178 }
11179
11180 switch (uVector)
11181 {
11182 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
11183 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
11184 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
11185 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
11186 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
11187 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
11188
11189 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11190 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11191 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11192 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11193 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11194 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11195 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11196 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11197 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11198 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11199 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11200 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11201 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11202 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11203 default:
11204 {
11205 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11206 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11207 {
11208 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11209 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11210 Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
11211
11212 rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0);
11213 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11214 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11215 AssertRCReturn(rc, rc);
11216 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11217 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11218 0 /* GCPtrFaultAddress */);
11219 }
11220 else
11221 {
11222 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11223 pVCpu->hm.s.u32HMError = uVector;
11224 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11225 }
11226 break;
11227 }
11228 }
11229 break;
11230 }
11231
11232 default:
11233 {
11234 pVCpu->hm.s.u32HMError = uExitIntInfo;
11235 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11236 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
11237 break;
11238 }
11239 }
11240 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11241 return rc;
11242}
11243
11244
11245/**
11246 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11247 */
11248HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11249{
11250 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11251
11252 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11253 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11254
11255 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11256 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11257 return VINF_SUCCESS;
11258}
11259
11260
11261/**
11262 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11263 */
11264HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11265{
11266 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11267 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)))
11268 {
11269 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11270 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11271 }
11272
11273 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11274
11275 /*
11276 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11277 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11278 */
11279 uint32_t fIntrState = 0;
11280 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
11281 AssertRCReturn(rc, rc);
11282
11283 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
11284 if ( fBlockSti
11285 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11286 {
11287 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11288 }
11289
11290 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11291 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11292
11293 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11294 return VINF_SUCCESS;
11295}
11296
11297
11298/**
11299 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11300 */
11301HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11302{
11303 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11304 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11305}
11306
11307
11308/**
11309 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11310 */
11311HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11312{
11313 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11314 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11315}
11316
11317
11318/**
11319 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11320 */
11321HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11322{
11323 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11324
11325 /*
11326 * Get the state we need and update the exit history entry.
11327 */
11328 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11329 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
11330 AssertRCReturn(rc, rc);
11331
11332 VBOXSTRICTRC rcStrict;
11333 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11334 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
11335 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11336 if (!pExitRec)
11337 {
11338 /*
11339 * Regular CPUID instruction execution.
11340 */
11341 rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
11342 if (rcStrict == VINF_SUCCESS)
11343 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11344 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11345 {
11346 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11347 rcStrict = VINF_SUCCESS;
11348 }
11349 }
11350 else
11351 {
11352 /*
11353 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
11354 */
11355 int rc2 = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11356 AssertRCReturn(rc2, rc2);
11357
11358 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11359 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11360
11361 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11362 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11363
11364 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11365 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11366 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11367 }
11368 return rcStrict;
11369}
11370
11371
11372/**
11373 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11374 */
11375HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11376{
11377 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11378 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR4);
11379 AssertRCReturn(rc, rc);
11380
11381 if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
11382 return VINF_EM_RAW_EMULATE_INSTR;
11383
11384 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11385 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11386}
11387
11388
11389/**
11390 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11391 */
11392HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11393{
11394 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11395 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11396 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11397 AssertRCReturn(rc, rc);
11398
11399 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11400 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11401 {
11402 /* If we get a spurious VM-exit when offsetting is enabled,
11403 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11404 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
11405 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11406 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11407 }
11408 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11409 {
11410 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11411 rcStrict = VINF_SUCCESS;
11412 }
11413 return rcStrict;
11414}
11415
11416
11417/**
11418 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11419 */
11420HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11421{
11422 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11423 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_TSC_AUX);
11424 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11425 AssertRCReturn(rc, rc);
11426
11427 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11428 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11429 {
11430 /* If we get a spurious VM-exit when offsetting is enabled,
11431 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11432 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
11433 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11434 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11435 }
11436 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11437 {
11438 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11439 rcStrict = VINF_SUCCESS;
11440 }
11441 return rcStrict;
11442}
11443
11444
11445/**
11446 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11447 */
11448HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11449{
11450 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11451 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11452 AssertRCReturn(rc, rc);
11453
11454 PVM pVM = pVCpu->CTX_SUFF(pVM);
11455 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11456 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11457 if (RT_LIKELY(rc == VINF_SUCCESS))
11458 {
11459 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11460 Assert(pVmxTransient->cbInstr == 2);
11461 }
11462 else
11463 {
11464 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11465 rc = VERR_EM_INTERPRETER;
11466 }
11467 return rc;
11468}
11469
11470
11471/**
11472 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11473 */
11474HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11475{
11476 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11477
11478 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11479 if (EMAreHypercallInstructionsEnabled(pVCpu))
11480 {
11481 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CR0
11482 | CPUMCTX_EXTRN_SS | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_EFER);
11483 AssertRCReturn(rc, rc);
11484
11485 /* Perform the hypercall. */
11486 rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
11487 if (rcStrict == VINF_SUCCESS)
11488 {
11489 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11490 AssertRCReturn(rc, rc);
11491 }
11492 else
11493 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11494 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11495 || RT_FAILURE(rcStrict));
11496
11497 /* If the hypercall changes anything other than guest's general-purpose registers,
11498 we would need to reload the guest changed bits here before VM-entry. */
11499 }
11500 else
11501 Log4Func(("Hypercalls not enabled\n"));
11502
11503 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11504 if (RT_FAILURE(rcStrict))
11505 {
11506 hmR0VmxSetPendingXcptUD(pVCpu);
11507 rcStrict = VINF_SUCCESS;
11508 }
11509
11510 return rcStrict;
11511}
11512
11513
11514/**
11515 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11516 */
11517HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11518{
11519 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11520 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11521
11522 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11523 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11524 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
11525 AssertRCReturn(rc, rc);
11526
11527 VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
11528
11529 if (rcStrict == VINF_SUCCESS || rcStrict == VINF_PGM_SYNC_CR3)
11530 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11531 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11532 {
11533 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11534 rcStrict = VINF_SUCCESS;
11535 }
11536 else
11537 AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) sttus: %Rrc\n", pVmxTransient->uExitQual,
11538 VBOXSTRICTRC_VAL(rcStrict)));
11539 return rcStrict;
11540}
11541
11542
11543/**
11544 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11545 */
11546HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11547{
11548 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11549 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11550 AssertRCReturn(rc, rc);
11551
11552 PVM pVM = pVCpu->CTX_SUFF(pVM);
11553 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11554 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11555 if (RT_LIKELY(rc == VINF_SUCCESS))
11556 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11557 else
11558 {
11559 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11560 rc = VERR_EM_INTERPRETER;
11561 }
11562 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11563 return rc;
11564}
11565
11566
11567/**
11568 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11569 */
11570HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11571{
11572 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11573 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11574 AssertRCReturn(rc, rc);
11575
11576 PVM pVM = pVCpu->CTX_SUFF(pVM);
11577 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11578 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11579 rc = VBOXSTRICTRC_VAL(rc2);
11580 if (RT_LIKELY( rc == VINF_SUCCESS
11581 || rc == VINF_EM_HALT))
11582 {
11583 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11584 AssertRCReturn(rc3, rc3);
11585
11586 if ( rc == VINF_EM_HALT
11587 && EMMonitorWaitShouldContinue(pVCpu, pCtx))
11588 rc = VINF_SUCCESS;
11589 }
11590 else
11591 {
11592 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11593 rc = VERR_EM_INTERPRETER;
11594 }
11595 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11596 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11597 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11598 return rc;
11599}
11600
11601
11602/**
11603 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11604 */
11605HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11606{
11607 /*
11608 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11609 * mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11610 * treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11611 * VMX root operation. If we get here, something funny is going on.
11612 *
11613 * See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11614 */
11615 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11616 AssertMsgFailed(("Unexpected RSM VM-exit\n"));
11617 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11618}
11619
11620
11621/**
11622 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11623 */
11624HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11625{
11626 /*
11627 * This can only happen if we support dual-monitor treatment of SMI, which can be activated
11628 * by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11629 * this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11630 * an SMI. If we get here, something funny is going on.
11631 *
11632 * See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11633 * See Intel spec. 25.3 "Other Causes of VM-Exits"
11634 */
11635 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11636 AssertMsgFailed(("Unexpected SMI VM-exit\n"));
11637 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11638}
11639
11640
11641/**
11642 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11643 */
11644HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11645{
11646 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11647 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11648 AssertMsgFailed(("Unexpected IO SMI VM-exit\n"));
11649 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11650}
11651
11652
11653/**
11654 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11655 */
11656HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11657{
11658 /*
11659 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11660 * We don't make use of it as our guests don't have direct access to the host LAPIC.
11661 * See Intel spec. 25.3 "Other Causes of VM-exits".
11662 */
11663 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11664 AssertMsgFailed(("Unexpected SIPI VM-exit\n"));
11665 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11666}
11667
11668
11669/**
11670 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11671 * VM-exit.
11672 */
11673HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11674{
11675 /*
11676 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11677 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11678 *
11679 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11680 * See Intel spec. "23.8 Restrictions on VMX operation".
11681 */
11682 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11683 return VINF_SUCCESS;
11684}
11685
11686
11687/**
11688 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11689 * VM-exit.
11690 */
11691HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11692{
11693 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11694 return VINF_EM_RESET;
11695}
11696
11697
11698/**
11699 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11700 */
11701HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11702{
11703 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11704 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_HLT_EXIT);
11705
11706 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11707 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RFLAGS);
11708 AssertRCReturn(rc, rc);
11709
11710 if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
11711 rc = VINF_SUCCESS;
11712 else
11713 rc = VINF_EM_HALT;
11714
11715 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11716 if (rc != VINF_SUCCESS)
11717 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11718 return rc;
11719}
11720
11721
11722/**
11723 * VM-exit handler for instructions that result in a \#UD exception delivered to
11724 * the guest.
11725 */
11726HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11727{
11728 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11729 hmR0VmxSetPendingXcptUD(pVCpu);
11730 return VINF_SUCCESS;
11731}
11732
11733
11734/**
11735 * VM-exit handler for expiry of the VMX preemption timer.
11736 */
11737HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11738{
11739 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11740
11741 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11742 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11743
11744 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11745 PVM pVM = pVCpu->CTX_SUFF(pVM);
11746 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11747 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11748 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11749}
11750
11751
11752/**
11753 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11754 */
11755HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11756{
11757 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11758
11759 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11760 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_CR4);
11761 AssertRCReturn(rc, rc);
11762
11763 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11764 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11765 : HM_CHANGED_RAISED_XCPT_MASK);
11766
11767 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11768 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
11769
11770 return rcStrict;
11771}
11772
11773
11774/**
11775 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11776 */
11777HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11778{
11779 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11780 /** @todo Use VM-exit instruction information. */
11781 return VERR_EM_INTERPRETER;
11782}
11783
11784
11785/**
11786 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11787 * Error VM-exit.
11788 */
11789HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11790{
11791 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11792 AssertRCReturn(rc, rc);
11793 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11794 if (RT_FAILURE(rc))
11795 return rc;
11796
11797 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
11798 NOREF(uInvalidReason);
11799
11800#ifdef VBOX_STRICT
11801 uint32_t fIntrState;
11802 RTHCUINTREG uHCReg;
11803 uint64_t u64Val;
11804 uint32_t u32Val;
11805
11806 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11807 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11808 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11809 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
11810 AssertRCReturn(rc, rc);
11811
11812 Log4(("uInvalidReason %u\n", uInvalidReason));
11813 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11814 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11815 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11816 Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
11817
11818 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11819 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11820 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11821 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11822 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11823 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11824 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11825 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11826 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11827 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11828 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11829 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11830
11831 hmR0DumpRegs(pVCpu);
11832#else
11833 NOREF(pVmxTransient);
11834#endif
11835
11836 return VERR_VMX_INVALID_GUEST_STATE;
11837}
11838
11839
11840/**
11841 * VM-exit handler for VM-entry failure due to an MSR-load
11842 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11843 */
11844HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11845{
11846 AssertMsgFailed(("Unexpected MSR-load exit\n"));
11847 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11848}
11849
11850
11851/**
11852 * VM-exit handler for VM-entry failure due to a machine-check event
11853 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11854 */
11855HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11856{
11857 AssertMsgFailed(("Unexpected machine-check event exit\n"));
11858 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11859}
11860
11861
11862/**
11863 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11864 * theory.
11865 */
11866HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11867{
11868 RT_NOREF2(pVCpu, pVmxTransient);
11869 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d\n", pVmxTransient->uExitReason));
11870 return VERR_VMX_UNDEFINED_EXIT_CODE;
11871}
11872
11873
11874/**
11875 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11876 * (VMX_EXIT_GDTR_IDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11877 * Conditional VM-exit.
11878 */
11879HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11880{
11881 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11882
11883 /* By default, we don't enable VMX_PROC_CTLS2_DESCRIPTOR_TABLE_EXIT. */
11884 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11885 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_DESC_TABLE_EXIT)
11886 return VERR_EM_INTERPRETER;
11887 AssertMsgFailed(("Unexpected XDTR access\n"));
11888 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11889}
11890
11891
11892/**
11893 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11894 */
11895HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11896{
11897 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11898
11899 /* By default, we don't enable VMX_PROC_CTLS2_RDRAND_EXIT. */
11900 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_RDRAND_EXIT)
11901 return VERR_EM_INTERPRETER;
11902 AssertMsgFailed(("Unexpected RDRAND exit\n"));
11903 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11904}
11905
11906
11907/**
11908 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11909 */
11910HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11911{
11912 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11913
11914 /** @todo Optimize this: We currently drag in in the whole MSR state
11915 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11916 * MSRs required. That would require changes to IEM and possibly CPUM too.
11917 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11918 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; NOREF(idMsr); /* Save it. */
11919 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11920 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
11921 AssertRCReturn(rc, rc);
11922
11923 Log4Func(("ecx=%#RX32\n", idMsr));
11924
11925#ifdef VBOX_STRICT
11926 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
11927 {
11928 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr)
11929 && idMsr != MSR_K6_EFER)
11930 {
11931 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
11932 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11933 }
11934 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11935 {
11936 VMXMSREXITREAD enmRead;
11937 VMXMSREXITWRITE enmWrite;
11938 int rc2 = HMVmxGetMsrPermission(pVCpu->hm.s.vmx.pvMsrBitmap, idMsr, &enmRead, &enmWrite);
11939 AssertRCReturn(rc2, rc2);
11940 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11941 {
11942 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
11943 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11944 }
11945 }
11946 }
11947#endif
11948
11949 VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
11950 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11951 if (rcStrict == VINF_SUCCESS)
11952 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11953 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX);
11954 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11955 {
11956 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11957 rcStrict = VINF_SUCCESS;
11958 }
11959 else
11960 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11961
11962 return rcStrict;
11963}
11964
11965
11966/**
11967 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11968 */
11969HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11970{
11971 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11972
11973 /** @todo Optimize this: We currently drag in in the whole MSR state
11974 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11975 * MSRs required. That would require changes to IEM and possibly CPUM too.
11976 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11977 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; /* Save it. */
11978 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11979 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
11980 AssertRCReturn(rc, rc);
11981
11982 Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
11983
11984 VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
11985 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
11986
11987 if (rcStrict == VINF_SUCCESS)
11988 {
11989 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11990
11991 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
11992 if ( idMsr == MSR_IA32_APICBASE
11993 || ( idMsr >= MSR_IA32_X2APIC_START
11994 && idMsr <= MSR_IA32_X2APIC_END))
11995 {
11996 /*
11997 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
11998 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before IEM changes it.
11999 */
12000 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
12001 }
12002 else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12003 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12004 else if (idMsr == MSR_K6_EFER)
12005 {
12006 /*
12007 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12008 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12009 * the other bits as well, SCE and NXE. See @bugref{7368}.
12010 */
12011 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS
12012 | HM_CHANGED_VMX_EXIT_CTLS);
12013 }
12014
12015 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12016 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
12017 {
12018 switch (idMsr)
12019 {
12020 case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
12021 case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
12022 case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
12023 case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
12024 case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
12025 case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
12026 default:
12027 {
12028 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
12029 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12030 else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
12031 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
12032 break;
12033 }
12034 }
12035 }
12036#ifdef VBOX_STRICT
12037 else
12038 {
12039 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12040 switch (idMsr)
12041 {
12042 case MSR_IA32_SYSENTER_CS:
12043 case MSR_IA32_SYSENTER_EIP:
12044 case MSR_IA32_SYSENTER_ESP:
12045 case MSR_K8_FS_BASE:
12046 case MSR_K8_GS_BASE:
12047 {
12048 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
12049 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12050 }
12051
12052 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12053 default:
12054 {
12055 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
12056 {
12057 /* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
12058 if (idMsr != MSR_K6_EFER)
12059 {
12060 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12061 idMsr));
12062 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12063 }
12064 }
12065
12066 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
12067 {
12068 VMXMSREXITREAD enmRead;
12069 VMXMSREXITWRITE enmWrite;
12070 int rc2 = HMVmxGetMsrPermission(pVCpu->hm.s.vmx.pvMsrBitmap, idMsr, &enmRead, &enmWrite);
12071 AssertRCReturn(rc2, rc2);
12072 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12073 {
12074 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
12075 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12076 }
12077 }
12078 break;
12079 }
12080 }
12081 }
12082#endif /* VBOX_STRICT */
12083 }
12084 else if (rcStrict == VINF_IEM_RAISED_XCPT)
12085 {
12086 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
12087 rcStrict = VINF_SUCCESS;
12088 }
12089 else
12090 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12091
12092 return rcStrict;
12093}
12094
12095
12096/**
12097 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12098 */
12099HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12100{
12101 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12102 /** @todo The guest has likely hit a contended spinlock. We might want to
12103 * poke a schedule different guest VCPU. */
12104 return VINF_EM_RAW_INTERRUPT;
12105}
12106
12107
12108/**
12109 * VM-exit handler for when the TPR value is lowered below the specified
12110 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12111 */
12112HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12113{
12114 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12115 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
12116
12117 /*
12118 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12119 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12120 */
12121 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12122 return VINF_SUCCESS;
12123}
12124
12125
12126/**
12127 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12128 * VM-exit.
12129 *
12130 * @retval VINF_SUCCESS when guest execution can continue.
12131 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12132 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12133 * interpreter.
12134 */
12135HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12136{
12137 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12138 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12139
12140 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12141 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12142 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12143 AssertRCReturn(rc, rc);
12144
12145 VBOXSTRICTRC rcStrict;
12146 PVM pVM = pVCpu->CTX_SUFF(pVM);
12147 RTGCUINTPTR const uExitQual = pVmxTransient->uExitQual;
12148 uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
12149 switch (uAccessType)
12150 {
12151 case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
12152 {
12153 uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
12154 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
12155 VMX_EXIT_QUAL_CRX_GENREG(uExitQual));
12156 AssertMsg( rcStrict == VINF_SUCCESS
12157 || rcStrict == VINF_IEM_RAISED_XCPT
12158 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12159
12160 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
12161 {
12162 case 0:
12163 {
12164 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12165 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12166 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
12167 Log4Func(("CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
12168
12169 /*
12170 * This is a kludge for handling switches back to real mode when we try to use
12171 * V86 mode to run real mode code directly. Problem is that V86 mode cannot
12172 * deal with special selector values, so we have to return to ring-3 and run
12173 * there till the selector values are V86 mode compatible.
12174 *
12175 * Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
12176 * latter is an alias for VINF_IEM_RAISED_XCPT which is converted to VINF_SUCCESs
12177 * at the end of this function.
12178 */
12179 if ( rc == VINF_SUCCESS
12180 && !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
12181 && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
12182 && (uOldCr0 & X86_CR0_PE)
12183 && !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
12184 {
12185 /** @todo check selectors rather than returning all the time. */
12186 Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
12187 rcStrict = VINF_EM_RESCHEDULE_REM;
12188 }
12189 break;
12190 }
12191
12192 case 2:
12193 {
12194 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
12195 /* Nothing to do here, CR2 it's not part of the VMCS. */
12196 break;
12197 }
12198
12199 case 3:
12200 {
12201 Assert( !pVM->hm.s.fNestedPaging
12202 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
12203 || pVCpu->hm.s.fUsingDebugLoop);
12204 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
12205 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12206 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR3);
12207 Log4Func(("CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
12208 break;
12209 }
12210
12211 case 4:
12212 {
12213 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
12214 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12215 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR4);
12216 Log4Func(("CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
12217 pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12218 break;
12219 }
12220
12221 case 8:
12222 {
12223 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
12224 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
12225 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12226 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_APIC_TPR);
12227 break;
12228 }
12229 default:
12230 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual)));
12231 break;
12232 }
12233 break;
12234 }
12235
12236 case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
12237 {
12238 Assert( !pVM->hm.s.fNestedPaging
12239 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
12240 || pVCpu->hm.s.fUsingDebugLoop
12241 || VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 3);
12242 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12243 Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 8
12244 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
12245
12246 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_GENREG(uExitQual),
12247 VMX_EXIT_QUAL_CRX_REGISTER(uExitQual));
12248 AssertMsg( rcStrict == VINF_SUCCESS
12249 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12250#ifdef VBOX_WITH_STATISTICS
12251 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
12252 {
12253 case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
12254 case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
12255 case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
12256 case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
12257 case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
12258 }
12259#endif
12260 Log4Func(("CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
12261 VBOXSTRICTRC_VAL(rcStrict)));
12262 if (VMX_EXIT_QUAL_CRX_GENREG(uExitQual) == X86_GREG_xSP)
12263 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RSP);
12264 else
12265 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12266 break;
12267 }
12268
12269 case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12270 {
12271 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12272 AssertMsg( rcStrict == VINF_SUCCESS
12273 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12274
12275 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12276 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12277 Log4Func(("CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12278 break;
12279 }
12280
12281 case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12282 {
12283 /* Note! LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here. */
12284 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual));
12285 AssertMsg( rcStrict == VINF_SUCCESS
12286 || rcStrict == VINF_IEM_RAISED_XCPT
12287 , ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12288
12289 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12290 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12291 Log4Func(("LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12292 break;
12293 }
12294
12295 default:
12296 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12297 VERR_VMX_UNEXPECTED_EXCEPTION);
12298 }
12299
12300 Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS))
12301 == (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS));
12302 if (rcStrict == VINF_IEM_RAISED_XCPT)
12303 {
12304 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
12305 rcStrict = VINF_SUCCESS;
12306 }
12307
12308 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12309 NOREF(pVM);
12310 return rcStrict;
12311}
12312
12313
12314/**
12315 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12316 * VM-exit.
12317 */
12318HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12319{
12320 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12321 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12322
12323 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12324 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12325 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12326 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_EFER);
12327 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12328 AssertRCReturn(rc, rc);
12329
12330 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12331 uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
12332 uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQual);
12333 bool fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12334 bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
12335 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
12336 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12337 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12338
12339 /*
12340 * Update exit history to see if this exit can be optimized.
12341 */
12342 VBOXSTRICTRC rcStrict;
12343 PCEMEXITREC pExitRec = NULL;
12344 if ( !fGstStepping
12345 && !fDbgStepping)
12346 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12347 !fIOString
12348 ? !fIOWrite
12349 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12350 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12351 : !fIOWrite
12352 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12353 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12354 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12355 if (!pExitRec)
12356 {
12357 /* I/O operation lookup arrays. */
12358 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12359 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12360 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12361 uint32_t const cbInstr = pVmxTransient->cbInstr;
12362 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12363 PVM pVM = pVCpu->CTX_SUFF(pVM);
12364 if (fIOString)
12365 {
12366 /*
12367 * INS/OUTS - I/O String instruction.
12368 *
12369 * Use instruction-information if available, otherwise fall back on
12370 * interpreting the instruction.
12371 */
12372 Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12373 AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
12374 bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
12375 if (fInsOutsInfo)
12376 {
12377 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12378 AssertRCReturn(rc2, rc2);
12379 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12380 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12381 IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12382 bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
12383 if (fIOWrite)
12384 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12385 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12386 else
12387 {
12388 /*
12389 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12390 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12391 * See Intel Instruction spec. for "INS".
12392 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12393 */
12394 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12395 }
12396 }
12397 else
12398 rcStrict = IEMExecOne(pVCpu);
12399
12400 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12401 fUpdateRipAlready = true;
12402 }
12403 else
12404 {
12405 /*
12406 * IN/OUT - I/O instruction.
12407 */
12408 Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12409 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12410 Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
12411 if (fIOWrite)
12412 {
12413 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
12414 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12415 if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
12416 && !pCtx->eflags.Bits.u1TF)
12417 rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
12418 }
12419 else
12420 {
12421 uint32_t u32Result = 0;
12422 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12423 if (IOM_SUCCESS(rcStrict))
12424 {
12425 /* Save result of I/O IN instr. in AL/AX/EAX. */
12426 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12427 }
12428 if ( rcStrict == VINF_IOM_R3_IOPORT_READ
12429 && !pCtx->eflags.Bits.u1TF)
12430 rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
12431 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12432 }
12433 }
12434
12435 if (IOM_SUCCESS(rcStrict))
12436 {
12437 if (!fUpdateRipAlready)
12438 {
12439 hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
12440 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12441 }
12442
12443 /*
12444 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12445 * while booting Fedora 17 64-bit guest.
12446 *
12447 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12448 */
12449 if (fIOString)
12450 {
12451 /** @todo Single-step for INS/OUTS with REP prefix? */
12452 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12453 }
12454 else if ( !fDbgStepping
12455 && fGstStepping)
12456 {
12457 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12458 AssertRCReturn(rc, rc);
12459 }
12460
12461 /*
12462 * If any I/O breakpoints are armed, we need to check if one triggered
12463 * and take appropriate action.
12464 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12465 */
12466 rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_DR7);
12467 AssertRCReturn(rc, rc);
12468
12469 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12470 * execution engines about whether hyper BPs and such are pending. */
12471 uint32_t const uDr7 = pCtx->dr[7];
12472 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12473 && X86_DR7_ANY_RW_IO(uDr7)
12474 && (pCtx->cr4 & X86_CR4_DE))
12475 || DBGFBpIsHwIoArmed(pVM)))
12476 {
12477 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12478
12479 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12480 VMMRZCallRing3Disable(pVCpu);
12481 HM_DISABLE_PREEMPT(pVCpu);
12482
12483 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12484
12485 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
12486 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12487 {
12488 /* Raise #DB. */
12489 if (fIsGuestDbgActive)
12490 ASMSetDR6(pCtx->dr[6]);
12491 if (pCtx->dr[7] != uDr7)
12492 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR7;
12493
12494 hmR0VmxSetPendingXcptDB(pVCpu);
12495 }
12496 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12497 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12498 else if ( rcStrict2 != VINF_SUCCESS
12499 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12500 rcStrict = rcStrict2;
12501 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12502
12503 HM_RESTORE_PREEMPT();
12504 VMMRZCallRing3Enable(pVCpu);
12505 }
12506 }
12507
12508#ifdef VBOX_STRICT
12509 if ( rcStrict == VINF_IOM_R3_IOPORT_READ
12510 || rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
12511 Assert(!fIOWrite);
12512 else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
12513 || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
12514 || rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
12515 Assert(fIOWrite);
12516 else
12517 {
12518# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12519 * statuses, that the VMM device and some others may return. See
12520 * IOM_SUCCESS() for guidance. */
12521 AssertMsg( RT_FAILURE(rcStrict)
12522 || rcStrict == VINF_SUCCESS
12523 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12524 || rcStrict == VINF_EM_DBG_BREAKPOINT
12525 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12526 || rcStrict == VINF_EM_RAW_TO_R3
12527 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12528# endif
12529 }
12530#endif
12531 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12532 }
12533 else
12534 {
12535 /*
12536 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12537 */
12538 int rc2 = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12539 AssertRCReturn(rc2, rc2);
12540 STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12541 : fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12542 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12543 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12544 VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
12545 fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12546
12547 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12548 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12549
12550 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12551 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12552 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12553 }
12554 return rcStrict;
12555}
12556
12557
12558/**
12559 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12560 * VM-exit.
12561 */
12562HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12563{
12564 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12565
12566 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12567 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12568 AssertRCReturn(rc, rc);
12569 if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12570 {
12571 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12572 AssertRCReturn(rc, rc);
12573 if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
12574 {
12575 uint32_t uErrCode;
12576 RTGCUINTPTR GCPtrFaultAddress;
12577 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12578 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12579 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
12580 if (fErrorCodeValid)
12581 {
12582 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12583 AssertRCReturn(rc, rc);
12584 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12585 }
12586 else
12587 uErrCode = 0;
12588
12589 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12590 && uVector == X86_XCPT_PF)
12591 GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
12592 else
12593 GCPtrFaultAddress = 0;
12594
12595 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12596 AssertRCReturn(rc, rc);
12597
12598 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12599 pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
12600
12601 Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
12602 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12603 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12604 }
12605 }
12606
12607 /* Fall back to the interpreter to emulate the task-switch. */
12608 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12609 return VERR_EM_INTERPRETER;
12610}
12611
12612
12613/**
12614 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12615 */
12616HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12617{
12618 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12619 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_MONITOR_TRAP_FLAG);
12620 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
12621 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12622 AssertRCReturn(rc, rc);
12623 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12624 return VINF_EM_DBG_STEPPED;
12625}
12626
12627
12628/**
12629 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12630 */
12631HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12632{
12633 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12634
12635 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12636
12637 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12638 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12639 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12640 {
12641 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12642 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12643 {
12644 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12645 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12646 }
12647 }
12648 else
12649 {
12650 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12651 rcStrict1 = VINF_SUCCESS;
12652 return rcStrict1;
12653 }
12654
12655 /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12656 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12657 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12658 AssertRCReturn(rc, rc);
12659
12660 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12661 uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
12662 VBOXSTRICTRC rcStrict2;
12663 switch (uAccessType)
12664 {
12665 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12666 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12667 {
12668 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
12669 || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
12670 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12671
12672 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12673 GCPhys &= PAGE_BASE_GC_MASK;
12674 GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
12675 PVM pVM = pVCpu->CTX_SUFF(pVM);
12676 Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12677 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
12678
12679 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12680 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12681 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12682 CPUMCTX2CORE(pCtx), GCPhys);
12683 Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12684 if ( rcStrict2 == VINF_SUCCESS
12685 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12686 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12687 {
12688 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12689 | HM_CHANGED_GUEST_APIC_TPR);
12690 rcStrict2 = VINF_SUCCESS;
12691 }
12692 break;
12693 }
12694
12695 default:
12696 Log4Func(("uAccessType=%#x\n", uAccessType));
12697 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12698 break;
12699 }
12700
12701 if (rcStrict2 != VINF_SUCCESS)
12702 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12703 return rcStrict2;
12704}
12705
12706
12707/**
12708 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12709 * VM-exit.
12710 */
12711HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12712{
12713 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12714
12715 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12716 if (pVmxTransient->fWasGuestDebugStateActive)
12717 {
12718 AssertMsgFailed(("Unexpected MOV DRx exit\n"));
12719 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12720 }
12721
12722 if ( !pVCpu->hm.s.fSingleInstruction
12723 && !pVmxTransient->fWasHyperDebugStateActive)
12724 {
12725 Assert(!DBGFIsStepping(pVCpu));
12726 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12727
12728 /* Don't intercept MOV DRx any more. */
12729 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
12730 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12731 AssertRCReturn(rc, rc);
12732
12733 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12734 VMMRZCallRing3Disable(pVCpu);
12735 HM_DISABLE_PREEMPT(pVCpu);
12736
12737 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12738 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12739 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12740
12741 HM_RESTORE_PREEMPT();
12742 VMMRZCallRing3Enable(pVCpu);
12743
12744#ifdef VBOX_WITH_STATISTICS
12745 rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12746 AssertRCReturn(rc, rc);
12747 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12748 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12749 else
12750 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12751#endif
12752 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12753 return VINF_SUCCESS;
12754 }
12755
12756 /*
12757 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12758 * Update the segment registers and DR7 from the CPU.
12759 */
12760 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12761 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12762 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_DR7);
12763 AssertRCReturn(rc, rc);
12764 Log4Func(("CS:RIP=%04x:%08RX64\n", pCtx->cs.Sel, pCtx->rip));
12765
12766 PVM pVM = pVCpu->CTX_SUFF(pVM);
12767 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12768 {
12769 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12770 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
12771 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
12772 if (RT_SUCCESS(rc))
12773 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12774 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12775 }
12776 else
12777 {
12778 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12779 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
12780 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
12781 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12782 }
12783
12784 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12785 if (RT_SUCCESS(rc))
12786 {
12787 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12788 AssertRCReturn(rc2, rc2);
12789 return VINF_SUCCESS;
12790 }
12791 return rc;
12792}
12793
12794
12795/**
12796 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12797 * Conditional VM-exit.
12798 */
12799HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12800{
12801 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12802 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12803
12804 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12805 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12806 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12807 {
12808 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12809 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12810 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12811 {
12812 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12813 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12814 }
12815 }
12816 else
12817 {
12818 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12819 rcStrict1 = VINF_SUCCESS;
12820 return rcStrict1;
12821 }
12822
12823 /*
12824 * Get sufficent state and update the exit history entry.
12825 */
12826 RTGCPHYS GCPhys;
12827 int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
12828 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12829 AssertRCReturn(rc, rc);
12830
12831 VBOXSTRICTRC rcStrict;
12832 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12833 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
12834 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12835 if (!pExitRec)
12836 {
12837 /*
12838 * If we succeed, resume guest execution.
12839 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12840 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12841 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12842 * weird case. See @bugref{6043}.
12843 */
12844 PVM pVM = pVCpu->CTX_SUFF(pVM);
12845 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12846 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
12847 Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12848 if ( rcStrict == VINF_SUCCESS
12849 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12850 || rcStrict == VERR_PAGE_NOT_PRESENT)
12851 {
12852 /* Successfully handled MMIO operation. */
12853 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12854 | HM_CHANGED_GUEST_APIC_TPR);
12855 rcStrict = VINF_SUCCESS;
12856 }
12857 }
12858 else
12859 {
12860 /*
12861 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12862 */
12863 int rc2 = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12864 AssertRCReturn(rc2, rc2);
12865
12866 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12867 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12868
12869 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12870 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12871
12872 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12873 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12874 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12875 }
12876 return VBOXSTRICTRC_TODO(rcStrict);
12877}
12878
12879
12880/**
12881 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12882 * VM-exit.
12883 */
12884HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12885{
12886 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12887 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12888
12889 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12890 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12891 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12892 {
12893 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12894 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12895 Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12896 }
12897 else
12898 {
12899 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12900 rcStrict1 = VINF_SUCCESS;
12901 return rcStrict1;
12902 }
12903
12904 RTGCPHYS GCPhys;
12905 int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
12906 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12907 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12908 AssertRCReturn(rc, rc);
12909
12910 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12911 AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
12912
12913 RTGCUINT uErrorCode = 0;
12914 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12915 uErrorCode |= X86_TRAP_PF_ID;
12916 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12917 uErrorCode |= X86_TRAP_PF_RW;
12918 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12919 uErrorCode |= X86_TRAP_PF_P;
12920
12921 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12922
12923
12924 /* Handle the pagefault trap for the nested shadow table. */
12925 PVM pVM = pVCpu->CTX_SUFF(pVM);
12926 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12927
12928 Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
12929 pCtx->cs.Sel, pCtx->rip));
12930
12931 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
12932 TRPMResetTrap(pVCpu);
12933
12934 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12935 if ( rcStrict2 == VINF_SUCCESS
12936 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12937 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12938 {
12939 /* Successfully synced our nested page tables. */
12940 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12941 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
12942 return VINF_SUCCESS;
12943 }
12944
12945 Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12946 return rcStrict2;
12947}
12948
12949/** @} */
12950
12951/** @name VM-exit exception handlers.
12952 * @{
12953 */
12954/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12955/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12956/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12957
12958/**
12959 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12960 */
12961static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12962{
12963 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12964 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12965
12966 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0);
12967 AssertRCReturn(rc, rc);
12968
12969 if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
12970 {
12971 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12972 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12973
12974 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12975 * provides VM-exit instruction length. If this causes problem later,
12976 * disassemble the instruction like it's done on AMD-V. */
12977 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12978 AssertRCReturn(rc2, rc2);
12979 return rc;
12980 }
12981
12982 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
12983 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12984 return rc;
12985}
12986
12987
12988/**
12989 * VM-exit exception handler for \#BP (Breakpoint exception).
12990 */
12991static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12992{
12993 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12994 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12995
12996 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12997 AssertRCReturn(rc, rc);
12998
12999 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13000 rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13001 if (rc == VINF_EM_RAW_GUEST_TRAP)
13002 {
13003 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13004 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13005 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13006 AssertRCReturn(rc, rc);
13007
13008 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13009 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13010 }
13011
13012 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
13013 return rc;
13014}
13015
13016
13017/**
13018 * VM-exit exception handler for \#AC (alignment check exception).
13019 */
13020static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13021{
13022 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13023
13024 /*
13025 * Re-inject it. We'll detect any nesting before getting here.
13026 */
13027 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13028 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13029 AssertRCReturn(rc, rc);
13030 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13031
13032 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13033 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13034 return VINF_SUCCESS;
13035}
13036
13037
13038/**
13039 * VM-exit exception handler for \#DB (Debug exception).
13040 */
13041static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13042{
13043 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13044 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13045
13046 /*
13047 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13048 * for processing.
13049 */
13050 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13051
13052 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13053 uint64_t uDR6 = X86_DR6_INIT_VAL;
13054 uDR6 |= (pVmxTransient->uExitQual & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
13055
13056 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13057 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13058 Log6Func(("rc=%Rrc\n", rc));
13059 if (rc == VINF_EM_RAW_GUEST_TRAP)
13060 {
13061 /*
13062 * The exception was for the guest. Update DR6, DR7.GD and
13063 * IA32_DEBUGCTL.LBR before forwarding it.
13064 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13065 */
13066 VMMRZCallRing3Disable(pVCpu);
13067 HM_DISABLE_PREEMPT(pVCpu);
13068
13069 pCtx->dr[6] &= ~X86_DR6_B_MASK;
13070 pCtx->dr[6] |= uDR6;
13071 if (CPUMIsGuestDebugStateActive(pVCpu))
13072 ASMSetDR6(pCtx->dr[6]);
13073
13074 HM_RESTORE_PREEMPT();
13075 VMMRZCallRing3Enable(pVCpu);
13076
13077 rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_DR7);
13078 AssertRCReturn(rc, rc);
13079
13080 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13081 pCtx->dr[7] &= ~X86_DR7_GD;
13082
13083 /* Paranoia. */
13084 pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13085 pCtx->dr[7] |= X86_DR7_RA1_MASK;
13086
13087 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
13088 AssertRCReturn(rc, rc);
13089
13090 /*
13091 * Raise #DB in the guest.
13092 *
13093 * It is important to reflect exactly what the VM-exit gave us (preserving the
13094 * interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
13095 * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
13096 * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
13097 *
13098 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
13099 * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
13100 */
13101 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13102 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13103 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13104 AssertRCReturn(rc, rc);
13105 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13106 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13107 return VINF_SUCCESS;
13108 }
13109
13110 /*
13111 * Not a guest trap, must be a hypervisor related debug event then.
13112 * Update DR6 in case someone is interested in it.
13113 */
13114 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13115 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13116 CPUMSetHyperDR6(pVCpu, uDR6);
13117
13118 return rc;
13119}
13120
13121
13122/**
13123 * VM-exit exception handler for \#GP (General-protection exception).
13124 *
13125 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13126 */
13127static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13128{
13129 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13130 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13131
13132 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13133 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13134 { /* likely */ }
13135 else
13136 {
13137#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13138 Assert(pVCpu->hm.s.fUsingDebugLoop);
13139#endif
13140 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13141 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13142 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13143 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13144 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13145 AssertRCReturn(rc, rc);
13146 Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pCtx->cs.Sel, pCtx->rip,
13147 pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
13148 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13149 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13150 return rc;
13151 }
13152
13153 Assert(CPUMIsGuestInRealModeEx(pCtx));
13154 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13155
13156 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13157 AssertRCReturn(rc, rc);
13158
13159 VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
13160 if (rcStrict == VINF_SUCCESS)
13161 {
13162 if (!CPUMIsGuestInRealModeEx(pCtx))
13163 {
13164 /*
13165 * The guest is no longer in real-mode, check if we can continue executing the
13166 * guest using hardware-assisted VMX. Otherwise, fall back to emulation.
13167 */
13168 if (HMVmxCanExecuteGuest(pVCpu, pCtx))
13169 {
13170 Log4Func(("Mode changed but guest still suitable for executing using VT-x\n"));
13171 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
13172 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13173 }
13174 else
13175 {
13176 Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
13177 rcStrict = VINF_EM_RESCHEDULE;
13178 }
13179 }
13180 else
13181 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13182 }
13183 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13184 {
13185 rcStrict = VINF_SUCCESS;
13186 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13187 }
13188 return VBOXSTRICTRC_VAL(rcStrict);
13189}
13190
13191
13192/**
13193 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13194 * the exception reported in the VMX transient structure back into the VM.
13195 *
13196 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13197 * up-to-date.
13198 */
13199static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13200{
13201 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13202#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13203 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13204 ("uVector=%#x u32XcptBitmap=%#X32\n",
13205 VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13206#endif
13207
13208 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13209 hmR0VmxCheckExitDueToEventDelivery(). */
13210 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13211 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13212 AssertRCReturn(rc, rc);
13213 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13214
13215#ifdef DEBUG_ramshankar
13216 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
13217 uint8_t uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13218 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13219#endif
13220
13221 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13222 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13223 return VINF_SUCCESS;
13224}
13225
13226
13227/**
13228 * VM-exit exception handler for \#PF (Page-fault exception).
13229 */
13230static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13231{
13232 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13233 PVM pVM = pVCpu->CTX_SUFF(pVM);
13234 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13235 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13236 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13237 AssertRCReturn(rc, rc);
13238
13239 if (!pVM->hm.s.fNestedPaging)
13240 { /* likely */ }
13241 else
13242 {
13243#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13244 Assert(pVCpu->hm.s.fUsingDebugLoop);
13245#endif
13246 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13247 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13248 {
13249 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13250 pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
13251 }
13252 else
13253 {
13254 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13255 hmR0VmxSetPendingXcptDF(pVCpu);
13256 Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13257 }
13258 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13259 return rc;
13260 }
13261
13262 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13263 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13264 if (pVmxTransient->fVectoringPF)
13265 {
13266 Assert(pVCpu->hm.s.Event.fPending);
13267 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13268 }
13269
13270 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13271 rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13272 AssertRCReturn(rc, rc);
13273
13274 Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
13275 pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
13276
13277 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13278 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
13279
13280 Log4Func(("#PF: rc=%Rrc\n", rc));
13281 if (rc == VINF_SUCCESS)
13282 {
13283 /*
13284 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13285 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13286 */
13287 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13288 TRPMResetTrap(pVCpu);
13289 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13290 return rc;
13291 }
13292
13293 if (rc == VINF_EM_RAW_GUEST_TRAP)
13294 {
13295 if (!pVmxTransient->fVectoringDoublePF)
13296 {
13297 /* It's a guest page fault and needs to be reflected to the guest. */
13298 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13299 TRPMResetTrap(pVCpu);
13300 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13301 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13302 uGstErrorCode, pVmxTransient->uExitQual);
13303 }
13304 else
13305 {
13306 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13307 TRPMResetTrap(pVCpu);
13308 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13309 hmR0VmxSetPendingXcptDF(pVCpu);
13310 Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13311 }
13312
13313 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13314 return VINF_SUCCESS;
13315 }
13316
13317 TRPMResetTrap(pVCpu);
13318 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13319 return rc;
13320}
13321
13322/** @} */
13323
13324#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13325/** @name Nested-guest VM-exit handlers.
13326 * @{
13327 */
13328/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13329/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= Nested-guest VM-exit handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13330/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13331
13332/**
13333 * VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
13334 */
13335HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13336{
13337 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13338
13339 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13340 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13341 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13342 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13343 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13344 AssertRCReturn(rc, rc);
13345
13346 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13347
13348 VMXVEXITINFO ExitInfo;
13349 RT_ZERO(ExitInfo);
13350 ExitInfo.uReason = pVmxTransient->uExitReason;
13351 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13352 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13353 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13354 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13355
13356 VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
13357 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13358 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13359 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13360 {
13361 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13362 rcStrict = VINF_SUCCESS;
13363 }
13364 return rcStrict;
13365}
13366
13367
13368/**
13369 * VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
13370 */
13371HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13372{
13373 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13374
13375 /** @todo NSTVMX: Vmlaunch. */
13376 hmR0VmxSetPendingXcptUD(pVCpu);
13377 return VINF_SUCCESS;
13378}
13379
13380
13381/**
13382 * VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
13383 */
13384HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13385{
13386 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13387
13388 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13389 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13390 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13391 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13392 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13393 AssertRCReturn(rc, rc);
13394
13395 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13396
13397 VMXVEXITINFO ExitInfo;
13398 RT_ZERO(ExitInfo);
13399 ExitInfo.uReason = pVmxTransient->uExitReason;
13400 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13401 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13402 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13403 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13404
13405 VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
13406 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13407 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13408 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13409 {
13410 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13411 rcStrict = VINF_SUCCESS;
13412 }
13413 return rcStrict;
13414}
13415
13416
13417/**
13418 * VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
13419 */
13420HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13421{
13422 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13423
13424 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13425 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13426 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13427 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13428 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13429 AssertRCReturn(rc, rc);
13430
13431 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13432
13433 VMXVEXITINFO ExitInfo;
13434 RT_ZERO(ExitInfo);
13435 ExitInfo.uReason = pVmxTransient->uExitReason;
13436 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13437 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13438 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13439 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
13440
13441 VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
13442 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13443 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13444 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13445 {
13446 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13447 rcStrict = VINF_SUCCESS;
13448 }
13449 return rcStrict;
13450}
13451
13452
13453/**
13454 * VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Unconditional VM-exit.
13455 */
13456HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13457{
13458 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13459
13460 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13461 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13462 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13463 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13464 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13465 AssertRCReturn(rc, rc);
13466
13467 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13468
13469 VMXVEXITINFO ExitInfo;
13470 RT_ZERO(ExitInfo);
13471 ExitInfo.uReason = pVmxTransient->uExitReason;
13472 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13473 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13474 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13475 if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
13476 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
13477
13478 VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
13479 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13480 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13481 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13482 {
13483 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13484 rcStrict = VINF_SUCCESS;
13485 }
13486 return rcStrict;
13487}
13488
13489
13490/**
13491 * VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
13492 */
13493HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13494{
13495 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13496
13497 /** @todo NSTVMX: Vmresume. */
13498 hmR0VmxSetPendingXcptUD(pVCpu);
13499 return VINF_SUCCESS;
13500}
13501
13502
13503/**
13504 * VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Unconditional VM-exit.
13505 */
13506HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13507{
13508 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13509
13510 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13511 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13512 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13513 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13514 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13515 AssertRCReturn(rc, rc);
13516
13517 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13518
13519 VMXVEXITINFO ExitInfo;
13520 RT_ZERO(ExitInfo);
13521 ExitInfo.uReason = pVmxTransient->uExitReason;
13522 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13523 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13524 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13525 if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
13526 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13527
13528 VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
13529 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13530 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13531 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13532 {
13533 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13534 rcStrict = VINF_SUCCESS;
13535 }
13536 return rcStrict;
13537}
13538
13539
13540/**
13541 * VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
13542 */
13543HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13544{
13545 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13546
13547 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13548 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13549 AssertRCReturn(rc, rc);
13550
13551 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13552
13553 VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
13554 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13555 {
13556 /* VMXOFF on success changes the internal hwvirt state but not anything that's visible to the guest. */
13557 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
13558 }
13559 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13560 {
13561 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13562 rcStrict = VINF_SUCCESS;
13563 }
13564 return rcStrict;
13565}
13566
13567
13568/**
13569 * VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
13570 */
13571HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13572{
13573 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13574
13575 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13576 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13577 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13578 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13579 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13580 AssertRCReturn(rc, rc);
13581
13582 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13583
13584 VMXVEXITINFO ExitInfo;
13585 RT_ZERO(ExitInfo);
13586 ExitInfo.uReason = pVmxTransient->uExitReason;
13587 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13588 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13589 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13590 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13591
13592 VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
13593 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13594 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13595 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13596 {
13597 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13598 rcStrict = VINF_SUCCESS;
13599 }
13600 return rcStrict;
13601}
13602
13603/** @} */
13604#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13605
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette