VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 74307

Last change on this file since 74307 was 74307, checked in by vboxsync, 6 years ago

VMM/HMVMXR0: Use a macro while importing guest-state from VM-exit handlers. Could be useful for debugging missed state synchronization between IEM and VMX R0 code.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 575.7 KB
Line 
1/* $Id: HMVMXR0.cpp 74307 2018-09-17 12:41:15Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/x86.h>
25#include <iprt/asm-amd64-x86.h>
26#include <iprt/thread.h>
27
28#include <VBox/vmm/pdmapi.h>
29#include <VBox/vmm/dbgf.h>
30#include <VBox/vmm/iem.h>
31#include <VBox/vmm/iom.h>
32#include <VBox/vmm/selm.h>
33#include <VBox/vmm/tm.h>
34#include <VBox/vmm/em.h>
35#include <VBox/vmm/gim.h>
36#include <VBox/vmm/apic.h>
37#ifdef VBOX_WITH_REM
38# include <VBox/vmm/rem.h>
39#endif
40#include "HMInternal.h"
41#include <VBox/vmm/vm.h>
42#include "HMVMXR0.h"
43#include "dtrace/VBoxVMM.h"
44
45#ifdef DEBUG_ramshankar
46# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
47# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
49# define HMVMX_ALWAYS_CHECK_GUEST_STATE
50# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
51# define HMVMX_ALWAYS_TRAP_PF
52# define HMVMX_ALWAYS_FLUSH_TLB
53# define HMVMX_ALWAYS_SWAP_EFER
54#endif
55
56
57/*********************************************************************************************************************************
58* Defined Constants And Macros *
59*********************************************************************************************************************************/
60/** Use the function table. */
61#define HMVMX_USE_FUNCTION_TABLE
62
63/** Determine which tagged-TLB flush handler to use. */
64#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
65#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
66#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
67#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
68
69/** @name HMVMX_READ_XXX
70 * Flags to skip redundant reads of some common VMCS fields that are not part of
71 * the guest-CPU or VCPU state but are needed while handling VM-exits.
72 */
73#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
74#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
75#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
76#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
77#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
78#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
79#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
80/** @} */
81
82/**
83 * States of the VMCS.
84 *
85 * This does not reflect all possible VMCS states but currently only those
86 * needed for maintaining the VMCS consistently even when thread-context hooks
87 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
88 */
89#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
90#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
91#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
92
93/**
94 * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
95 * guest using hardware-assisted VMX.
96 *
97 * This excludes state like GPRs (other than RSP) which are always are
98 * swapped and restored across the world-switch and also registers like EFER,
99 * MSR which cannot be modified by the guest without causing a VM-exit.
100 */
101#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
102 | CPUMCTX_EXTRN_RFLAGS \
103 | CPUMCTX_EXTRN_RSP \
104 | CPUMCTX_EXTRN_SREG_MASK \
105 | CPUMCTX_EXTRN_TABLE_MASK \
106 | CPUMCTX_EXTRN_KERNEL_GS_BASE \
107 | CPUMCTX_EXTRN_SYSCALL_MSRS \
108 | CPUMCTX_EXTRN_SYSENTER_MSRS \
109 | CPUMCTX_EXTRN_TSC_AUX \
110 | CPUMCTX_EXTRN_OTHER_MSRS \
111 | CPUMCTX_EXTRN_CR0 \
112 | CPUMCTX_EXTRN_CR3 \
113 | CPUMCTX_EXTRN_CR4 \
114 | CPUMCTX_EXTRN_DR7 \
115 | CPUMCTX_EXTRN_HM_VMX_MASK)
116
117/**
118 * Exception bitmap mask for real-mode guests (real-on-v86).
119 *
120 * We need to intercept all exceptions manually except:
121 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
122 * due to bugs in Intel CPUs.
123 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
124 * support.
125 */
126#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
127 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
128 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
129 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
130 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
131 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
132 | RT_BIT(X86_XCPT_XF))
133
134/** Maximum VM-instruction error number. */
135#define HMVMX_INSTR_ERROR_MAX 28
136
137/** Profiling macro. */
138#ifdef HM_PROFILE_EXIT_DISPATCH
139# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
140# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
141#else
142# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
143# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
144#endif
145
146/** Assert that preemption is disabled or covered by thread-context hooks. */
147#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
148 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
149
150/** Assert that we haven't migrated CPUs when thread-context hooks are not
151 * used. */
152#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
153 || (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
154 ("Illegal migration! Entered on CPU %u Current %u\n", \
155 (a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
156
157/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
158 * context. */
159#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
160 ("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
161 (a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
162
163/** Macro for importing guest state from the VMCS back into CPUMCTX (intended to be
164 * used only from VM-exit handlers). */
165#define HMVMX_CPUMCTX_IMPORT_STATE(a_pVCpu, a_fWhat) (hmR0VmxImportGuestState((a_pVCpu), (a_fWhat)))
166
167/** Helper macro for VM-exit handlers called unexpectedly. */
168#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
169 do { \
170 (a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
171 return VERR_VMX_UNEXPECTED_EXIT; \
172 } while (0)
173
174/** Macro for importing segment registers to the VMCS from the guest-CPU context. */
175#ifdef VMX_USE_CACHED_VMCS_ACCESSES
176# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
177 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
178 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
179#else
180# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
181 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
182 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
183#endif
184
185/** Macro for exporting segment registers to the VMCS from the guest-CPU context. */
186#define HMVMX_EXPORT_SREG(Sel, a_pCtxSelReg) \
187 hmR0VmxExportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
188 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
189
190#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
191/** Macro that does the necessary privilege checks and intercepted VM-exits for
192 * guests that attempted to execute a VMX instruction. */
193# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
194 do \
195 { \
196 VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
197 if (rcStrictTmp == VINF_SUCCESS) \
198 { /* likely */ } \
199 else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
200 { \
201 Assert((a_pVCpu)->hm.s.Event.fPending); \
202 Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
203 return VINF_SUCCESS; \
204 } \
205 else \
206 { \
207 int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
208 AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
209 } \
210 } while (0)
211
212/** Macro that decodes a memory operand for an instruction VM-exit. */
213# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
214 do \
215 { \
216 VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
217 (a_pGCPtrEffAddr)); \
218 if (rcStrictTmp == VINF_SUCCESS) \
219 { /* likely */ } \
220 else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
221 { \
222 uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
223 Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
224 return VINF_SUCCESS; \
225 } \
226 else \
227 { \
228 Log4Func(("hmR0VmxCheckExitDueToVmxInstr failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
229 return rcStrictTmp; \
230 } \
231 } while (0)
232
233#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
234
235
236/*********************************************************************************************************************************
237* Structures and Typedefs *
238*********************************************************************************************************************************/
239/**
240 * VMX transient state.
241 *
242 * A state structure for holding miscellaneous information across
243 * VMX non-root operation and restored after the transition.
244 */
245typedef struct VMXTRANSIENT
246{
247 /** The host's rflags/eflags. */
248 RTCCUINTREG fEFlags;
249#if HC_ARCH_BITS == 32
250 uint32_t u32Alignment0;
251#endif
252 /** The guest's TPR value used for TPR shadowing. */
253 uint8_t u8GuestTpr;
254 /** Alignment. */
255 uint8_t abAlignment0[7];
256
257 /** The basic VM-exit reason. */
258 uint16_t uExitReason;
259 /** Alignment. */
260 uint16_t u16Alignment0;
261 /** The VM-exit interruption error code. */
262 uint32_t uExitIntErrorCode;
263 /** The VM-exit exit code qualification. */
264 uint64_t uExitQual;
265
266 /** The VM-exit interruption-information field. */
267 uint32_t uExitIntInfo;
268 /** The VM-exit instruction-length field. */
269 uint32_t cbInstr;
270 /** The VM-exit instruction-information field. */
271 VMXEXITINSTRINFO ExitInstrInfo;
272 /** Whether the VM-entry failed or not. */
273 bool fVMEntryFailed;
274 /** Alignment. */
275 uint8_t abAlignment1[3];
276
277 /** The VM-entry interruption-information field. */
278 uint32_t uEntryIntInfo;
279 /** The VM-entry exception error code field. */
280 uint32_t uEntryXcptErrorCode;
281 /** The VM-entry instruction length field. */
282 uint32_t cbEntryInstr;
283
284 /** IDT-vectoring information field. */
285 uint32_t uIdtVectoringInfo;
286 /** IDT-vectoring error code. */
287 uint32_t uIdtVectoringErrorCode;
288
289 /** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
290 uint32_t fVmcsFieldsRead;
291
292 /** Whether the guest debug state was active at the time of VM-exit. */
293 bool fWasGuestDebugStateActive;
294 /** Whether the hyper debug state was active at the time of VM-exit. */
295 bool fWasHyperDebugStateActive;
296 /** Whether TSC-offsetting should be setup before VM-entry. */
297 bool fUpdateTscOffsettingAndPreemptTimer;
298 /** Whether the VM-exit was caused by a page-fault during delivery of a
299 * contributory exception or a page-fault. */
300 bool fVectoringDoublePF;
301 /** Whether the VM-exit was caused by a page-fault during delivery of an
302 * external interrupt or NMI. */
303 bool fVectoringPF;
304} VMXTRANSIENT;
305AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
306AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
307AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
308AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
309AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
310/** Pointer to VMX transient state. */
311typedef VMXTRANSIENT *PVMXTRANSIENT;
312
313/**
314 * MSR-bitmap read permissions.
315 */
316typedef enum VMXMSREXITREAD
317{
318 /** Reading this MSR causes a VM-exit. */
319 VMXMSREXIT_INTERCEPT_READ = 0xb,
320 /** Reading this MSR does not cause a VM-exit. */
321 VMXMSREXIT_PASSTHRU_READ
322} VMXMSREXITREAD;
323/** Pointer to MSR-bitmap read permissions. */
324typedef VMXMSREXITREAD* PVMXMSREXITREAD;
325
326/**
327 * MSR-bitmap write permissions.
328 */
329typedef enum VMXMSREXITWRITE
330{
331 /** Writing to this MSR causes a VM-exit. */
332 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
333 /** Writing to this MSR does not cause a VM-exit. */
334 VMXMSREXIT_PASSTHRU_WRITE
335} VMXMSREXITWRITE;
336/** Pointer to MSR-bitmap write permissions. */
337typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
338
339/**
340 * Memory operand read or write access.
341 */
342typedef enum VMXMEMACCESS
343{
344 VMXMEMACCESS_READ = 0,
345 VMXMEMACCESS_WRITE = 1
346} VMXMEMACCESS;
347
348/**
349 * VMX VM-exit handler.
350 *
351 * @returns Strict VBox status code (i.e. informational status codes too).
352 * @param pVCpu The cross context virtual CPU structure.
353 * @param pVmxTransient Pointer to the VMX-transient structure.
354 */
355#ifndef HMVMX_USE_FUNCTION_TABLE
356typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
357#else
358typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
359/** Pointer to VM-exit handler. */
360typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
361#endif
362
363/**
364 * VMX VM-exit handler, non-strict status code.
365 *
366 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
367 *
368 * @returns VBox status code, no informational status code returned.
369 * @param pVCpu The cross context virtual CPU structure.
370 * @param pVmxTransient Pointer to the VMX-transient structure.
371 *
372 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
373 * use of that status code will be replaced with VINF_EM_SOMETHING
374 * later when switching over to IEM.
375 */
376#ifndef HMVMX_USE_FUNCTION_TABLE
377typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
378#else
379typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
380#endif
381
382
383/*********************************************************************************************************************************
384* Internal Functions *
385*********************************************************************************************************************************/
386static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush);
387static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr);
388static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
389static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
390static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
391 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
392#if HC_ARCH_BITS == 32
393static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
394#endif
395#ifndef HMVMX_USE_FUNCTION_TABLE
396DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
397# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
398# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
399#else
400# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
401# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
402#endif
403
404/** @name VM-exit handlers.
405 * @{
406 */
407static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
408static FNVMXEXITHANDLER hmR0VmxExitExtInt;
409static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
410static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
411static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
412static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
413static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
414static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
415static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
416static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
417static FNVMXEXITHANDLER hmR0VmxExitCpuid;
418static FNVMXEXITHANDLER hmR0VmxExitGetsec;
419static FNVMXEXITHANDLER hmR0VmxExitHlt;
420static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
421static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
422static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
423static FNVMXEXITHANDLER hmR0VmxExitVmcall;
424#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
425static FNVMXEXITHANDLER hmR0VmxExitVmclear;
426static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
427static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
428static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
429static FNVMXEXITHANDLER hmR0VmxExitVmread;
430static FNVMXEXITHANDLER hmR0VmxExitVmresume;
431static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
432static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
433static FNVMXEXITHANDLER hmR0VmxExitVmxon;
434#endif
435static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
436static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
437static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
438static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
439static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
440static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
441static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
442static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
443static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
444static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
445static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
446static FNVMXEXITHANDLER hmR0VmxExitMwait;
447static FNVMXEXITHANDLER hmR0VmxExitMtf;
448static FNVMXEXITHANDLER hmR0VmxExitMonitor;
449static FNVMXEXITHANDLER hmR0VmxExitPause;
450static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
451static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
452static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
453static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
454static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
455static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
456static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
457static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
458static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
459static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
460static FNVMXEXITHANDLER hmR0VmxExitRdrand;
461static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
462/** @} */
463
464static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
465static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
466static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
467static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
468static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
469static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
470static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
471static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu);
472
473
474/*********************************************************************************************************************************
475* Global Variables *
476*********************************************************************************************************************************/
477#ifdef HMVMX_USE_FUNCTION_TABLE
478
479/**
480 * VMX_EXIT dispatch table.
481 */
482static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
483{
484 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
485 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
486 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
487 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
488 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
489 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
490 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
491 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
492 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
493 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
494 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
495 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
496 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
497 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
498 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
499 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
500 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
501 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
502 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
503#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
504 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
505 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
506 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
507 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
508 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
509 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
510 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
511 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
512 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
513#else
514 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
515 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
516 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
517 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
518 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
519 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
520 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
521 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
522 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
523#endif
524 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
525 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
526 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
527 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
528 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
529 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
530 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
531 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
532 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
533 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
534 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
535 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
536 /* 40 UNDEFINED */ hmR0VmxExitPause,
537 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
538 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
539 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
540 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
541 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
542 /* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitXdtrAccess,
543 /* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitXdtrAccess,
544 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
545 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
546 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
547 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
548 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
549 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
550 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
551 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
552 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
553 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
554 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
555 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
556 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
557 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
558 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
559 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
560 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
561};
562#endif /* HMVMX_USE_FUNCTION_TABLE */
563
564#ifdef VBOX_STRICT
565static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
566{
567 /* 0 */ "(Not Used)",
568 /* 1 */ "VMCALL executed in VMX root operation.",
569 /* 2 */ "VMCLEAR with invalid physical address.",
570 /* 3 */ "VMCLEAR with VMXON pointer.",
571 /* 4 */ "VMLAUNCH with non-clear VMCS.",
572 /* 5 */ "VMRESUME with non-launched VMCS.",
573 /* 6 */ "VMRESUME after VMXOFF",
574 /* 7 */ "VM-entry with invalid control fields.",
575 /* 8 */ "VM-entry with invalid host state fields.",
576 /* 9 */ "VMPTRLD with invalid physical address.",
577 /* 10 */ "VMPTRLD with VMXON pointer.",
578 /* 11 */ "VMPTRLD with incorrect revision identifier.",
579 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
580 /* 13 */ "VMWRITE to read-only VMCS component.",
581 /* 14 */ "(Not Used)",
582 /* 15 */ "VMXON executed in VMX root operation.",
583 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
584 /* 17 */ "VM-entry with non-launched executing VMCS.",
585 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
586 /* 19 */ "VMCALL with non-clear VMCS.",
587 /* 20 */ "VMCALL with invalid VM-exit control fields.",
588 /* 21 */ "(Not Used)",
589 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
590 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
591 /* 24 */ "VMCALL with invalid SMM-monitor features.",
592 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
593 /* 26 */ "VM-entry with events blocked by MOV SS.",
594 /* 27 */ "(Not Used)",
595 /* 28 */ "Invalid operand to INVEPT/INVVPID."
596};
597#endif /* VBOX_STRICT */
598
599
600/**
601 * Updates the VM's last error record.
602 *
603 * If there was a VMX instruction error, reads the error data from the VMCS and
604 * updates VCPU's last error record as well.
605 *
606 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
607 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
608 * VERR_VMX_INVALID_VMCS_FIELD.
609 * @param rc The error code.
610 */
611static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
612{
613 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
614 || rc == VERR_VMX_UNABLE_TO_START_VM)
615 {
616 AssertPtrReturnVoid(pVCpu);
617 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
618 }
619 pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
620}
621
622
623/**
624 * Reads the VM-entry interruption-information field from the VMCS into the VMX
625 * transient structure.
626 *
627 * @returns VBox status code.
628 * @param pVmxTransient Pointer to the VMX transient structure.
629 *
630 * @remarks No-long-jump zone!!!
631 */
632DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
633{
634 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
635 AssertRCReturn(rc, rc);
636 return VINF_SUCCESS;
637}
638
639#ifdef VBOX_STRICT
640/**
641 * Reads the VM-entry exception error code field from the VMCS into
642 * the VMX transient structure.
643 *
644 * @returns VBox status code.
645 * @param pVmxTransient Pointer to the VMX transient structure.
646 *
647 * @remarks No-long-jump zone!!!
648 */
649DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
650{
651 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
652 AssertRCReturn(rc, rc);
653 return VINF_SUCCESS;
654}
655
656
657/**
658 * Reads the VM-entry exception error code field from the VMCS into
659 * the VMX transient structure.
660 *
661 * @returns VBox status code.
662 * @param pVmxTransient Pointer to the VMX transient structure.
663 *
664 * @remarks No-long-jump zone!!!
665 */
666DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
667{
668 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
669 AssertRCReturn(rc, rc);
670 return VINF_SUCCESS;
671}
672#endif /* VBOX_STRICT */
673
674
675/**
676 * Reads the VM-exit interruption-information field from the VMCS into the VMX
677 * transient structure.
678 *
679 * @returns VBox status code.
680 * @param pVmxTransient Pointer to the VMX transient structure.
681 */
682DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
683{
684 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
685 {
686 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
687 AssertRCReturn(rc,rc);
688 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_INFO;
689 }
690 return VINF_SUCCESS;
691}
692
693
694/**
695 * Reads the VM-exit interruption error code from the VMCS into the VMX
696 * transient structure.
697 *
698 * @returns VBox status code.
699 * @param pVmxTransient Pointer to the VMX transient structure.
700 */
701DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
702{
703 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
704 {
705 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
706 AssertRCReturn(rc, rc);
707 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
708 }
709 return VINF_SUCCESS;
710}
711
712
713/**
714 * Reads the VM-exit instruction length field from the VMCS into the VMX
715 * transient structure.
716 *
717 * @returns VBox status code.
718 * @param pVmxTransient Pointer to the VMX transient structure.
719 */
720DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
721{
722 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
723 {
724 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
725 AssertRCReturn(rc, rc);
726 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_LEN;
727 }
728 return VINF_SUCCESS;
729}
730
731
732/**
733 * Reads the VM-exit instruction-information field from the VMCS into
734 * the VMX transient structure.
735 *
736 * @returns VBox status code.
737 * @param pVmxTransient Pointer to the VMX transient structure.
738 */
739DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
740{
741 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
742 {
743 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
744 AssertRCReturn(rc, rc);
745 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_INFO;
746 }
747 return VINF_SUCCESS;
748}
749
750
751/**
752 * Reads the exit code qualification from the VMCS into the VMX transient
753 * structure.
754 *
755 * @returns VBox status code.
756 * @param pVCpu The cross context virtual CPU structure of the
757 * calling EMT. (Required for the VMCS cache case.)
758 * @param pVmxTransient Pointer to the VMX transient structure.
759 */
760DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
761{
762 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
763 {
764 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
765 AssertRCReturn(rc, rc);
766 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION;
767 }
768 return VINF_SUCCESS;
769}
770
771
772/**
773 * Reads the IDT-vectoring information field from the VMCS into the VMX
774 * transient structure.
775 *
776 * @returns VBox status code.
777 * @param pVmxTransient Pointer to the VMX transient structure.
778 *
779 * @remarks No-long-jump zone!!!
780 */
781DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
782{
783 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
784 {
785 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
786 AssertRCReturn(rc, rc);
787 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_INFO;
788 }
789 return VINF_SUCCESS;
790}
791
792
793/**
794 * Reads the IDT-vectoring error code from the VMCS into the VMX
795 * transient structure.
796 *
797 * @returns VBox status code.
798 * @param pVmxTransient Pointer to the VMX transient structure.
799 */
800DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
801{
802 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
803 {
804 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
805 AssertRCReturn(rc, rc);
806 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
807 }
808 return VINF_SUCCESS;
809}
810
811
812/**
813 * Enters VMX root mode operation on the current CPU.
814 *
815 * @returns VBox status code.
816 * @param pVM The cross context VM structure. Can be
817 * NULL, after a resume.
818 * @param HCPhysCpuPage Physical address of the VMXON region.
819 * @param pvCpuPage Pointer to the VMXON region.
820 */
821static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
822{
823 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
824 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
825 Assert(pvCpuPage);
826 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
827
828 if (pVM)
829 {
830 /* Write the VMCS revision dword to the VMXON region. */
831 *(uint32_t *)pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
832 }
833
834 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
835 RTCCUINTREG fEFlags = ASMIntDisableFlags();
836
837 /* Enable the VMX bit in CR4 if necessary. */
838 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
839
840 /* Enter VMX root mode. */
841 int rc = VMXEnable(HCPhysCpuPage);
842 if (RT_FAILURE(rc))
843 {
844 if (!(uOldCr4 & X86_CR4_VMXE))
845 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
846
847 if (pVM)
848 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
849 }
850
851 /* Restore interrupts. */
852 ASMSetFlags(fEFlags);
853 return rc;
854}
855
856
857/**
858 * Exits VMX root mode operation on the current CPU.
859 *
860 * @returns VBox status code.
861 */
862static int hmR0VmxLeaveRootMode(void)
863{
864 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
865
866 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
867 RTCCUINTREG fEFlags = ASMIntDisableFlags();
868
869 /* If we're for some reason not in VMX root mode, then don't leave it. */
870 RTCCUINTREG uHostCR4 = ASMGetCR4();
871
872 int rc;
873 if (uHostCR4 & X86_CR4_VMXE)
874 {
875 /* Exit VMX root mode and clear the VMX bit in CR4. */
876 VMXDisable();
877 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
878 rc = VINF_SUCCESS;
879 }
880 else
881 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
882
883 /* Restore interrupts. */
884 ASMSetFlags(fEFlags);
885 return rc;
886}
887
888
889/**
890 * Allocates and maps one physically contiguous page. The allocated page is
891 * zero'd out. (Used by various VT-x structures).
892 *
893 * @returns IPRT status code.
894 * @param pMemObj Pointer to the ring-0 memory object.
895 * @param ppVirt Where to store the virtual address of the
896 * allocation.
897 * @param pHCPhys Where to store the physical address of the
898 * allocation.
899 */
900static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
901{
902 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
903 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
904 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
905
906 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
907 if (RT_FAILURE(rc))
908 return rc;
909 *ppVirt = RTR0MemObjAddress(*pMemObj);
910 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
911 ASMMemZero32(*ppVirt, PAGE_SIZE);
912 return VINF_SUCCESS;
913}
914
915
916/**
917 * Frees and unmaps an allocated physical page.
918 *
919 * @param pMemObj Pointer to the ring-0 memory object.
920 * @param ppVirt Where to re-initialize the virtual address of
921 * allocation as 0.
922 * @param pHCPhys Where to re-initialize the physical address of the
923 * allocation as 0.
924 */
925static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
926{
927 AssertPtr(pMemObj);
928 AssertPtr(ppVirt);
929 AssertPtr(pHCPhys);
930 if (*pMemObj != NIL_RTR0MEMOBJ)
931 {
932 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
933 AssertRC(rc);
934 *pMemObj = NIL_RTR0MEMOBJ;
935 *ppVirt = 0;
936 *pHCPhys = 0;
937 }
938}
939
940
941/**
942 * Worker function to free VT-x related structures.
943 *
944 * @returns IPRT status code.
945 * @param pVM The cross context VM structure.
946 */
947static void hmR0VmxStructsFree(PVM pVM)
948{
949 for (VMCPUID i = 0; i < pVM->cCpus; i++)
950 {
951 PVMCPU pVCpu = &pVM->aCpus[i];
952 AssertPtr(pVCpu);
953
954 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
955 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
956
957 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
958 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
959
960 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
961 }
962
963 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
964#ifdef VBOX_WITH_CRASHDUMP_MAGIC
965 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
966#endif
967}
968
969
970/**
971 * Worker function to allocate VT-x related VM structures.
972 *
973 * @returns IPRT status code.
974 * @param pVM The cross context VM structure.
975 */
976static int hmR0VmxStructsAlloc(PVM pVM)
977{
978 /*
979 * Initialize members up-front so we can cleanup properly on allocation failure.
980 */
981#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
982 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
983 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
984 pVM->hm.s.vmx.HCPhys##a_Name = 0;
985
986#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
987 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
988 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
989 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
990
991#ifdef VBOX_WITH_CRASHDUMP_MAGIC
992 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
993#endif
994 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
995
996 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
997 for (VMCPUID i = 0; i < pVM->cCpus; i++)
998 {
999 PVMCPU pVCpu = &pVM->aCpus[i];
1000 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
1001 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
1002 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
1003 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
1004 }
1005#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
1006#undef VMXLOCAL_INIT_VM_MEMOBJ
1007
1008 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
1009 AssertReturnStmt(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE) <= PAGE_SIZE,
1010 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
1011 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
1012
1013 /*
1014 * Allocate all the VT-x structures.
1015 */
1016 int rc = VINF_SUCCESS;
1017#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1018 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1019 if (RT_FAILURE(rc))
1020 goto cleanup;
1021 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1022 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1023#endif
1024
1025 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1026 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1027 {
1028 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1029 &pVM->hm.s.vmx.HCPhysApicAccess);
1030 if (RT_FAILURE(rc))
1031 goto cleanup;
1032 }
1033
1034 /*
1035 * Initialize per-VCPU VT-x structures.
1036 */
1037 for (VMCPUID i = 0; i < pVM->cCpus; i++)
1038 {
1039 PVMCPU pVCpu = &pVM->aCpus[i];
1040 AssertPtr(pVCpu);
1041
1042 /* Allocate the VM control structure (VMCS). */
1043 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1044 if (RT_FAILURE(rc))
1045 goto cleanup;
1046
1047 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1048 if ( PDMHasApic(pVM)
1049 && (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1050 {
1051 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1052 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1053 if (RT_FAILURE(rc))
1054 goto cleanup;
1055 }
1056
1057 /*
1058 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1059 * transparent accesses of specific MSRs.
1060 *
1061 * If the condition for enabling MSR bitmaps changes here, don't forget to
1062 * update HMAreMsrBitmapsAvailable().
1063 */
1064 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1065 {
1066 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1067 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1068 if (RT_FAILURE(rc))
1069 goto cleanup;
1070 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1071 }
1072
1073 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1074 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1075 if (RT_FAILURE(rc))
1076 goto cleanup;
1077
1078 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1079 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1080 if (RT_FAILURE(rc))
1081 goto cleanup;
1082 }
1083
1084 return VINF_SUCCESS;
1085
1086cleanup:
1087 hmR0VmxStructsFree(pVM);
1088 return rc;
1089}
1090
1091
1092/**
1093 * Does global VT-x initialization (called during module initialization).
1094 *
1095 * @returns VBox status code.
1096 */
1097VMMR0DECL(int) VMXR0GlobalInit(void)
1098{
1099#ifdef HMVMX_USE_FUNCTION_TABLE
1100 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1101# ifdef VBOX_STRICT
1102 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1103 Assert(g_apfnVMExitHandlers[i]);
1104# endif
1105#endif
1106 return VINF_SUCCESS;
1107}
1108
1109
1110/**
1111 * Does global VT-x termination (called during module termination).
1112 */
1113VMMR0DECL(void) VMXR0GlobalTerm()
1114{
1115 /* Nothing to do currently. */
1116}
1117
1118
1119/**
1120 * Sets up and activates VT-x on the current CPU.
1121 *
1122 * @returns VBox status code.
1123 * @param pHostCpu Pointer to the global CPU info struct.
1124 * @param pVM The cross context VM structure. Can be
1125 * NULL after a host resume operation.
1126 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1127 * fEnabledByHost is @c true).
1128 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1129 * @a fEnabledByHost is @c true).
1130 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1131 * enable VT-x on the host.
1132 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1133 */
1134VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1135 void *pvMsrs)
1136{
1137 Assert(pHostCpu);
1138 Assert(pvMsrs);
1139 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1140
1141 /* Enable VT-x if it's not already enabled by the host. */
1142 if (!fEnabledByHost)
1143 {
1144 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1145 if (RT_FAILURE(rc))
1146 return rc;
1147 }
1148
1149 /*
1150 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1151 * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1152 * invalidated when flushing by VPID.
1153 */
1154 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1155 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1156 {
1157 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
1158 pHostCpu->fFlushAsidBeforeUse = false;
1159 }
1160 else
1161 pHostCpu->fFlushAsidBeforeUse = true;
1162
1163 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1164 ++pHostCpu->cTlbFlushes;
1165
1166 return VINF_SUCCESS;
1167}
1168
1169
1170/**
1171 * Deactivates VT-x on the current CPU.
1172 *
1173 * @returns VBox status code.
1174 * @param pHostCpu Pointer to the global CPU info struct.
1175 * @param pvCpuPage Pointer to the VMXON region.
1176 * @param HCPhysCpuPage Physical address of the VMXON region.
1177 *
1178 * @remarks This function should never be called when SUPR0EnableVTx() or
1179 * similar was used to enable VT-x on the host.
1180 */
1181VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1182{
1183 RT_NOREF3(pHostCpu, pvCpuPage, HCPhysCpuPage);
1184
1185 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1186 return hmR0VmxLeaveRootMode();
1187}
1188
1189
1190/**
1191 * Sets the permission bits for the specified MSR in the MSR bitmap.
1192 *
1193 * @param pVCpu The cross context virtual CPU structure.
1194 * @param uMsr The MSR value.
1195 * @param enmRead Whether reading this MSR causes a VM-exit.
1196 * @param enmWrite Whether writing this MSR causes a VM-exit.
1197 */
1198static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1199{
1200 int32_t iBit;
1201 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1202
1203 /*
1204 * Layout:
1205 * 0x000 - 0x3ff - Low MSR read bits
1206 * 0x400 - 0x7ff - High MSR read bits
1207 * 0x800 - 0xbff - Low MSR write bits
1208 * 0xc00 - 0xfff - High MSR write bits
1209 */
1210 if (uMsr <= 0x00001fff)
1211 iBit = uMsr;
1212 else if (uMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1213 {
1214 iBit = uMsr - UINT32_C(0xc0000000);
1215 pbMsrBitmap += 0x400;
1216 }
1217 else
1218 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1219
1220 Assert(iBit <= 0x1fff);
1221 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1222 ASMBitSet(pbMsrBitmap, iBit);
1223 else
1224 ASMBitClear(pbMsrBitmap, iBit);
1225
1226 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1227 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1228 else
1229 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1230}
1231
1232
1233#ifdef VBOX_STRICT
1234/**
1235 * Gets the permission bits for the specified MSR in the MSR bitmap.
1236 *
1237 * @returns VBox status code.
1238 * @retval VINF_SUCCESS if the specified MSR is found.
1239 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1240 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1241 *
1242 * @param pVCpu The cross context virtual CPU structure.
1243 * @param uMsr The MSR.
1244 * @param penmRead Where to store the read permissions.
1245 * @param penmWrite Where to store the write permissions.
1246 */
1247static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1248{
1249 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1250 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1251 int32_t iBit;
1252 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1253
1254 /* See hmR0VmxSetMsrPermission() for the layout. */
1255 if (uMsr <= 0x00001fff)
1256 iBit = uMsr;
1257 else if ( uMsr >= 0xc0000000
1258 && uMsr <= 0xc0001fff)
1259 {
1260 iBit = (uMsr - 0xc0000000);
1261 pbMsrBitmap += 0x400;
1262 }
1263 else
1264 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1265
1266 Assert(iBit <= 0x1fff);
1267 if (ASMBitTest(pbMsrBitmap, iBit))
1268 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1269 else
1270 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1271
1272 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1273 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1274 else
1275 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1276 return VINF_SUCCESS;
1277}
1278#endif /* VBOX_STRICT */
1279
1280
1281/**
1282 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1283 * area.
1284 *
1285 * @returns VBox status code.
1286 * @param pVCpu The cross context virtual CPU structure.
1287 * @param cMsrs The number of MSRs.
1288 */
1289static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1290{
1291 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1292 uint64_t const uVmxMiscMsr = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc;
1293 uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(uVmxMiscMsr);
1294 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1295 {
1296 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1297 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1298 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1299 }
1300
1301 /* Update number of guest MSRs to load/store across the world-switch. */
1302 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1303 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1304
1305 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1306 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1307 AssertRCReturn(rc, rc);
1308
1309 /* Update the VCPU's copy of the MSR count. */
1310 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1311
1312 return VINF_SUCCESS;
1313}
1314
1315
1316/**
1317 * Adds a new (or updates the value of an existing) guest/host MSR
1318 * pair to be swapped during the world-switch as part of the
1319 * auto-load/store MSR area in the VMCS.
1320 *
1321 * @returns VBox status code.
1322 * @param pVCpu The cross context virtual CPU structure.
1323 * @param uMsr The MSR.
1324 * @param uGuestMsrValue Value of the guest MSR.
1325 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1326 * necessary.
1327 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1328 * its value was updated. Optional, can be NULL.
1329 */
1330static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1331 bool *pfAddedAndUpdated)
1332{
1333 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1334 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1335 uint32_t i;
1336 for (i = 0; i < cMsrs; i++)
1337 {
1338 if (pGuestMsr->u32Msr == uMsr)
1339 break;
1340 pGuestMsr++;
1341 }
1342
1343 bool fAdded = false;
1344 if (i == cMsrs)
1345 {
1346 ++cMsrs;
1347 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1348 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1349
1350 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1351 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1352 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1353
1354 fAdded = true;
1355 }
1356
1357 /* Update the MSR values in the auto-load/store MSR area. */
1358 pGuestMsr->u32Msr = uMsr;
1359 pGuestMsr->u64Value = uGuestMsrValue;
1360
1361 /* Create/update the MSR slot in the host MSR area. */
1362 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1363 pHostMsr += i;
1364 pHostMsr->u32Msr = uMsr;
1365
1366 /*
1367 * Update the host MSR only when requested by the caller AND when we're
1368 * adding it to the auto-load/store area. Otherwise, it would have been
1369 * updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1370 */
1371 bool fUpdatedMsrValue = false;
1372 if ( fAdded
1373 && fUpdateHostMsr)
1374 {
1375 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1376 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1377 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1378 fUpdatedMsrValue = true;
1379 }
1380
1381 if (pfAddedAndUpdated)
1382 *pfAddedAndUpdated = fUpdatedMsrValue;
1383 return VINF_SUCCESS;
1384}
1385
1386
1387/**
1388 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1389 * auto-load/store MSR area in the VMCS.
1390 *
1391 * @returns VBox status code.
1392 * @param pVCpu The cross context virtual CPU structure.
1393 * @param uMsr The MSR.
1394 */
1395static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1396{
1397 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1398 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1399 for (uint32_t i = 0; i < cMsrs; i++)
1400 {
1401 /* Find the MSR. */
1402 if (pGuestMsr->u32Msr == uMsr)
1403 {
1404 /* If it's the last MSR, simply reduce the count. */
1405 if (i == cMsrs - 1)
1406 {
1407 --cMsrs;
1408 break;
1409 }
1410
1411 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1412 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1413 pLastGuestMsr += cMsrs - 1;
1414 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1415 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1416
1417 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1418 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1419 pLastHostMsr += cMsrs - 1;
1420 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1421 pHostMsr->u64Value = pLastHostMsr->u64Value;
1422 --cMsrs;
1423 break;
1424 }
1425 pGuestMsr++;
1426 }
1427
1428 /* Update the VMCS if the count changed (meaning the MSR was found). */
1429 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1430 {
1431 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1432 AssertRCReturn(rc, rc);
1433
1434 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1435 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1436 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1437
1438 Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1439 return VINF_SUCCESS;
1440 }
1441
1442 return VERR_NOT_FOUND;
1443}
1444
1445
1446/**
1447 * Checks if the specified guest MSR is part of the auto-load/store area in
1448 * the VMCS.
1449 *
1450 * @returns true if found, false otherwise.
1451 * @param pVCpu The cross context virtual CPU structure.
1452 * @param uMsr The MSR to find.
1453 */
1454static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1455{
1456 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1457 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1458
1459 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1460 {
1461 if (pGuestMsr->u32Msr == uMsr)
1462 return true;
1463 }
1464 return false;
1465}
1466
1467
1468/**
1469 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1470 *
1471 * @param pVCpu The cross context virtual CPU structure.
1472 *
1473 * @remarks No-long-jump zone!!!
1474 */
1475static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1476{
1477 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1478 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1479 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1480 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1481
1482 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1483 {
1484 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1485
1486 /*
1487 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1488 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1489 */
1490 if (pHostMsr->u32Msr == MSR_K6_EFER)
1491 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1492 else
1493 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1494 }
1495
1496 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1497}
1498
1499
1500/**
1501 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1502 * perform lazy restoration of the host MSRs while leaving VT-x.
1503 *
1504 * @param pVCpu The cross context virtual CPU structure.
1505 *
1506 * @remarks No-long-jump zone!!!
1507 */
1508static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1509{
1510 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1511
1512 /*
1513 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1514 */
1515 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1516 {
1517 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1518#if HC_ARCH_BITS == 64
1519 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1520 {
1521 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1522 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1523 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1524 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1525 }
1526#endif
1527 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1528 }
1529}
1530
1531
1532/**
1533 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1534 * lazily while leaving VT-x.
1535 *
1536 * @returns true if it does, false otherwise.
1537 * @param pVCpu The cross context virtual CPU structure.
1538 * @param uMsr The MSR to check.
1539 */
1540static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1541{
1542 NOREF(pVCpu);
1543#if HC_ARCH_BITS == 64
1544 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1545 {
1546 switch (uMsr)
1547 {
1548 case MSR_K8_LSTAR:
1549 case MSR_K6_STAR:
1550 case MSR_K8_SF_MASK:
1551 case MSR_K8_KERNEL_GS_BASE:
1552 return true;
1553 }
1554 }
1555#else
1556 RT_NOREF(pVCpu, uMsr);
1557#endif
1558 return false;
1559}
1560
1561
1562/**
1563 * Loads a set of guests MSRs to allow read/passthru to the guest.
1564 *
1565 * The name of this function is slightly confusing. This function does NOT
1566 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1567 * common prefix for functions dealing with "lazy restoration" of the shared
1568 * MSRs.
1569 *
1570 * @param pVCpu The cross context virtual CPU structure.
1571 *
1572 * @remarks No-long-jump zone!!!
1573 */
1574static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1575{
1576 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1577 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1578
1579 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1580#if HC_ARCH_BITS == 64
1581 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1582 {
1583 /*
1584 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1585 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1586 * we can skip a few MSR writes.
1587 *
1588 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1589 * guest MSR values in the guest-CPU context might be different to what's currently
1590 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1591 * CPU, see @bugref{8728}.
1592 */
1593 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1594 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1595 && pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1596 && pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1597 && pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1598 && pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1599 {
1600#ifdef VBOX_STRICT
1601 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1602 Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1603 Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1604 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
1605#endif
1606 }
1607 else
1608 {
1609 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1610 ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
1611 ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
1612 ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
1613 }
1614 }
1615#endif
1616 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1617}
1618
1619
1620/**
1621 * Performs lazy restoration of the set of host MSRs if they were previously
1622 * loaded with guest MSR values.
1623 *
1624 * @param pVCpu The cross context virtual CPU structure.
1625 *
1626 * @remarks No-long-jump zone!!!
1627 * @remarks The guest MSRs should have been saved back into the guest-CPU
1628 * context by hmR0VmxImportGuestState()!!!
1629 */
1630static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1631{
1632 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1633 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1634
1635 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1636 {
1637 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1638#if HC_ARCH_BITS == 64
1639 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1640 {
1641 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1642 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1643 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1644 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1645 }
1646#endif
1647 }
1648 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1649}
1650
1651
1652/**
1653 * Verifies that our cached values of the VMCS fields are all consistent with
1654 * what's actually present in the VMCS.
1655 *
1656 * @returns VBox status code.
1657 * @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
1658 * @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
1659 * VMCS content. HMCPU error-field is
1660 * updated, see VMX_VCI_XXX.
1661 * @param pVCpu The cross context virtual CPU structure.
1662 */
1663static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1664{
1665 uint32_t u32Val;
1666 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1667 AssertRCReturn(rc, rc);
1668 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32EntryCtls == u32Val,
1669 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1670 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
1671 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1672
1673 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1674 AssertRCReturn(rc, rc);
1675 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ExitCtls == u32Val,
1676 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1677 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
1678 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1679
1680 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1681 AssertRCReturn(rc, rc);
1682 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32PinCtls == u32Val,
1683 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1684 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
1685 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1686
1687 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1688 AssertRCReturn(rc, rc);
1689 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls == u32Val,
1690 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1691 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
1692 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1693
1694 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
1695 {
1696 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1697 AssertRCReturn(rc, rc);
1698 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1699 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1700 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
1701 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1702 }
1703
1704 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
1705 AssertRCReturn(rc, rc);
1706 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32XcptBitmap == u32Val,
1707 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap, u32Val),
1708 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
1709 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1710
1711 uint64_t u64Val;
1712 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
1713 AssertRCReturn(rc, rc);
1714 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u64TscOffset == u64Val,
1715 ("Cache=%#RX64 VMCS=%#RX64\n", pVCpu->hm.s.vmx.u64TscOffset, u64Val),
1716 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
1717 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1718
1719 return VINF_SUCCESS;
1720}
1721
1722
1723#ifdef VBOX_STRICT
1724/**
1725 * Verifies that our cached host EFER value has not changed
1726 * since we cached it.
1727 *
1728 * @param pVCpu The cross context virtual CPU structure.
1729 */
1730static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1731{
1732 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1733
1734 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
1735 {
1736 uint64_t u64Val;
1737 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1738 AssertRC(rc);
1739
1740 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1741 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1742 }
1743}
1744
1745
1746/**
1747 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1748 * VMCS are correct.
1749 *
1750 * @param pVCpu The cross context virtual CPU structure.
1751 */
1752static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1753{
1754 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1755
1756 /* Verify MSR counts in the VMCS are what we think it should be. */
1757 uint32_t cMsrs;
1758 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1759 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1760
1761 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1762 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1763
1764 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1765 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1766
1767 PCVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1768 PCVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1769 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1770 {
1771 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1772 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1773 pGuestMsr->u32Msr, cMsrs));
1774
1775 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1776 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1777 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1778
1779 /* Verify that the permissions are as expected in the MSR bitmap. */
1780 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1781 {
1782 VMXMSREXITREAD enmRead;
1783 VMXMSREXITWRITE enmWrite;
1784 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1785 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1786 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1787 {
1788 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1789 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1790 }
1791 else
1792 {
1793 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1794 pGuestMsr->u32Msr, cMsrs));
1795 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1796 pGuestMsr->u32Msr, cMsrs));
1797 }
1798 }
1799 }
1800}
1801#endif /* VBOX_STRICT */
1802
1803
1804/**
1805 * Flushes the TLB using EPT.
1806 *
1807 * @returns VBox status code.
1808 * @param pVCpu The cross context virtual CPU structure of the calling
1809 * EMT. Can be NULL depending on @a enmTlbFlush.
1810 * @param enmTlbFlush Type of flush.
1811 *
1812 * @remarks Caller is responsible for making sure this function is called only
1813 * when NestedPaging is supported and providing @a enmTlbFlush that is
1814 * supported by the CPU.
1815 * @remarks Can be called with interrupts disabled.
1816 */
1817static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush)
1818{
1819 uint64_t au64Descriptor[2];
1820 if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
1821 au64Descriptor[0] = 0;
1822 else
1823 {
1824 Assert(pVCpu);
1825 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1826 }
1827 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1828
1829 int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
1830 AssertMsg(rc == VINF_SUCCESS,
1831 ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0, rc));
1832
1833 if ( RT_SUCCESS(rc)
1834 && pVCpu)
1835 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1836}
1837
1838
1839/**
1840 * Flushes the TLB using VPID.
1841 *
1842 * @returns VBox status code.
1843 * @param pVCpu The cross context virtual CPU structure of the calling
1844 * EMT. Can be NULL depending on @a enmTlbFlush.
1845 * @param enmTlbFlush Type of flush.
1846 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1847 * on @a enmTlbFlush).
1848 *
1849 * @remarks Can be called with interrupts disabled.
1850 */
1851static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
1852{
1853 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
1854
1855 uint64_t au64Descriptor[2];
1856 if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
1857 {
1858 au64Descriptor[0] = 0;
1859 au64Descriptor[1] = 0;
1860 }
1861 else
1862 {
1863 AssertPtr(pVCpu);
1864 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1865 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1866 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1867 au64Descriptor[1] = GCPtr;
1868 }
1869
1870 int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
1871 AssertMsg(rc == VINF_SUCCESS,
1872 ("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1873
1874 if ( RT_SUCCESS(rc)
1875 && pVCpu)
1876 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1877 NOREF(rc);
1878}
1879
1880
1881/**
1882 * Invalidates a guest page by guest virtual address. Only relevant for
1883 * EPT/VPID, otherwise there is nothing really to invalidate.
1884 *
1885 * @returns VBox status code.
1886 * @param pVCpu The cross context virtual CPU structure.
1887 * @param GCVirt Guest virtual address of the page to invalidate.
1888 */
1889VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
1890{
1891 AssertPtr(pVCpu);
1892 LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
1893
1894 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1895 if (!fFlushPending)
1896 {
1897 /*
1898 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1899 * the EPT case. See @bugref{6043} and @bugref{6177}.
1900 *
1901 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1902 * as this function maybe called in a loop with individual addresses.
1903 */
1904 PVM pVM = pVCpu->CTX_SUFF(pVM);
1905 if (pVM->hm.s.vmx.fVpid)
1906 {
1907 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1908
1909#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1910 /*
1911 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1912 * where executing INVVPID outside 64-bit mode does not flush translations of
1913 * 64-bit linear addresses, see @bugref{6208#c72}.
1914 */
1915 if (RT_HI_U32(GCVirt))
1916 fVpidFlush = false;
1917#endif
1918
1919 if (fVpidFlush)
1920 {
1921 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
1922 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1923 }
1924 else
1925 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1926 }
1927 else if (pVM->hm.s.fNestedPaging)
1928 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1929 }
1930
1931 return VINF_SUCCESS;
1932}
1933
1934
1935/**
1936 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1937 * case where neither EPT nor VPID is supported by the CPU.
1938 *
1939 * @param pVCpu The cross context virtual CPU structure.
1940 * @param pCpu Pointer to the global HM struct.
1941 *
1942 * @remarks Called with interrupts disabled.
1943 */
1944static void hmR0VmxFlushTaggedTlbNone(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1945{
1946 AssertPtr(pVCpu);
1947 AssertPtr(pCpu);
1948
1949 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1950
1951 Assert(pCpu->idCpu != NIL_RTCPUID);
1952 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1953 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1954 pVCpu->hm.s.fForceTLBFlush = false;
1955 return;
1956}
1957
1958
1959/**
1960 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1961 *
1962 * @param pVCpu The cross context virtual CPU structure.
1963 * @param pCpu Pointer to the global HM CPU struct.
1964 *
1965 * @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
1966 * nomenclature. The reason is, to avoid confusion in compare statements
1967 * since the host-CPU copies are named "ASID".
1968 *
1969 * @remarks Called with interrupts disabled.
1970 */
1971static void hmR0VmxFlushTaggedTlbBoth(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1972{
1973#ifdef VBOX_WITH_STATISTICS
1974 bool fTlbFlushed = false;
1975# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1976# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1977 if (!fTlbFlushed) \
1978 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1979 } while (0)
1980#else
1981# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1982# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1983#endif
1984
1985 AssertPtr(pCpu);
1986 AssertPtr(pVCpu);
1987 Assert(pCpu->idCpu != NIL_RTCPUID);
1988
1989 PVM pVM = pVCpu->CTX_SUFF(pVM);
1990 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1991 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1992 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1993
1994 /*
1995 * Force a TLB flush for the first world-switch if the current CPU differs from the one we
1996 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1997 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1998 * cannot reuse the current ASID anymore.
1999 */
2000 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2001 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2002 {
2003 ++pCpu->uCurrentAsid;
2004 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2005 {
2006 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2007 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2008 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2009 }
2010
2011 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2012 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2013 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2014
2015 /*
2016 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2017 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2018 */
2019 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
2020 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2021 HMVMX_SET_TAGGED_TLB_FLUSHED();
2022 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2023 }
2024 else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2025 {
2026 /*
2027 * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2028 * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2029 * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2030 * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2031 * mappings, see @bugref{6568}.
2032 *
2033 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2034 */
2035 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
2036 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2037 HMVMX_SET_TAGGED_TLB_FLUSHED();
2038 }
2039
2040 pVCpu->hm.s.fForceTLBFlush = false;
2041 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2042
2043 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2044 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2045 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2046 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2047 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2048 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2049 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2050 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2051 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2052
2053 /* Update VMCS with the VPID. */
2054 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2055 AssertRC(rc);
2056
2057#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2058}
2059
2060
2061/**
2062 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2063 *
2064 * @returns VBox status code.
2065 * @param pVCpu The cross context virtual CPU structure.
2066 * @param pCpu Pointer to the global HM CPU struct.
2067 *
2068 * @remarks Called with interrupts disabled.
2069 */
2070static void hmR0VmxFlushTaggedTlbEpt(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2071{
2072 AssertPtr(pVCpu);
2073 AssertPtr(pCpu);
2074 Assert(pCpu->idCpu != NIL_RTCPUID);
2075 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2076 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2077
2078 /*
2079 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2080 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2081 */
2082 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2083 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2084 {
2085 pVCpu->hm.s.fForceTLBFlush = true;
2086 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2087 }
2088
2089 /* Check for explicit TLB flushes. */
2090 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2091 {
2092 pVCpu->hm.s.fForceTLBFlush = true;
2093 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2094 }
2095
2096 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2097 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2098
2099 if (pVCpu->hm.s.fForceTLBFlush)
2100 {
2101 hmR0VmxFlushEpt(pVCpu, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2102 pVCpu->hm.s.fForceTLBFlush = false;
2103 }
2104}
2105
2106
2107/**
2108 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2109 *
2110 * @returns VBox status code.
2111 * @param pVCpu The cross context virtual CPU structure.
2112 * @param pCpu Pointer to the global HM CPU struct.
2113 *
2114 * @remarks Called with interrupts disabled.
2115 */
2116static void hmR0VmxFlushTaggedTlbVpid(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2117{
2118 AssertPtr(pVCpu);
2119 AssertPtr(pCpu);
2120 Assert(pCpu->idCpu != NIL_RTCPUID);
2121 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2122 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2123
2124 /*
2125 * Force a TLB flush for the first world switch if the current CPU differs from the one we
2126 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2127 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2128 * cannot reuse the current ASID anymore.
2129 */
2130 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2131 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2132 {
2133 pVCpu->hm.s.fForceTLBFlush = true;
2134 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2135 }
2136
2137 /* Check for explicit TLB flushes. */
2138 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2139 {
2140 /*
2141 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2142 * hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2143 * fExplicitFlush = true here and change the pCpu->fFlushAsidBeforeUse check below to
2144 * include fExplicitFlush's too) - an obscure corner case.
2145 */
2146 pVCpu->hm.s.fForceTLBFlush = true;
2147 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2148 }
2149
2150 PVM pVM = pVCpu->CTX_SUFF(pVM);
2151 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2152 if (pVCpu->hm.s.fForceTLBFlush)
2153 {
2154 ++pCpu->uCurrentAsid;
2155 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2156 {
2157 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2158 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2159 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2160 }
2161
2162 pVCpu->hm.s.fForceTLBFlush = false;
2163 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2164 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2165 if (pCpu->fFlushAsidBeforeUse)
2166 {
2167 if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2168 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2169 else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2170 {
2171 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2172 pCpu->fFlushAsidBeforeUse = false;
2173 }
2174 else
2175 {
2176 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2177 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2178 }
2179 }
2180 }
2181
2182 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2183 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2184 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2185 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2186 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2187 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2188 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2189
2190 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2191 AssertRC(rc);
2192}
2193
2194
2195/**
2196 * Flushes the guest TLB entry based on CPU capabilities.
2197 *
2198 * @param pVCpu The cross context virtual CPU structure.
2199 * @param pCpu Pointer to the global HM CPU struct.
2200 */
2201DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2202{
2203#ifdef HMVMX_ALWAYS_FLUSH_TLB
2204 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2205#endif
2206 PVM pVM = pVCpu->CTX_SUFF(pVM);
2207 switch (pVM->hm.s.vmx.enmTlbFlushType)
2208 {
2209 case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVCpu, pCpu); break;
2210 case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pVCpu, pCpu); break;
2211 case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pVCpu, pCpu); break;
2212 case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pVCpu, pCpu); break;
2213 default:
2214 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2215 break;
2216 }
2217 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2218}
2219
2220
2221/**
2222 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2223 * TLB entries from the host TLB before VM-entry.
2224 *
2225 * @returns VBox status code.
2226 * @param pVM The cross context VM structure.
2227 */
2228static int hmR0VmxSetupTaggedTlb(PVM pVM)
2229{
2230 /*
2231 * Determine optimal flush type for Nested Paging.
2232 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2233 * guest execution (see hmR3InitFinalizeR0()).
2234 */
2235 if (pVM->hm.s.fNestedPaging)
2236 {
2237 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2238 {
2239 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2240 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2241 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2242 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2243 else
2244 {
2245 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2246 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2247 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2248 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2249 }
2250
2251 /* Make sure the write-back cacheable memory type for EPT is supported. */
2252 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2253 {
2254 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2255 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2256 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2257 }
2258
2259 /* EPT requires a page-walk length of 4. */
2260 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2261 {
2262 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2263 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2264 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2265 }
2266 }
2267 else
2268 {
2269 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2270 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2271 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2272 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2273 }
2274 }
2275
2276 /*
2277 * Determine optimal flush type for VPID.
2278 */
2279 if (pVM->hm.s.vmx.fVpid)
2280 {
2281 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2282 {
2283 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2284 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2285 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2286 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2287 else
2288 {
2289 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2290 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2291 LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2292 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2293 LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2294 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2295 pVM->hm.s.vmx.fVpid = false;
2296 }
2297 }
2298 else
2299 {
2300 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2301 Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2302 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2303 pVM->hm.s.vmx.fVpid = false;
2304 }
2305 }
2306
2307 /*
2308 * Setup the handler for flushing tagged-TLBs.
2309 */
2310 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2311 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2312 else if (pVM->hm.s.fNestedPaging)
2313 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2314 else if (pVM->hm.s.vmx.fVpid)
2315 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2316 else
2317 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2318 return VINF_SUCCESS;
2319}
2320
2321
2322/**
2323 * Sets up pin-based VM-execution controls in the VMCS.
2324 *
2325 * @returns VBox status code.
2326 * @param pVCpu The cross context virtual CPU structure.
2327 *
2328 * @remarks We don't really care about optimizing vmwrites here as it's done only
2329 * once per VM and hence we don't care about VMCS-field cache comparisons.
2330 */
2331static int hmR0VmxSetupPinCtls(PVMCPU pVCpu)
2332{
2333 PVM pVM = pVCpu->CTX_SUFF(pVM);
2334 uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.disallowed0; /* Bits set here must always be set. */
2335 uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2336
2337 fVal |= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2338 | VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2339
2340 if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2341 fVal |= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2342
2343 /* Enable the VMX preemption timer. */
2344 if (pVM->hm.s.vmx.fUsePreemptTimer)
2345 {
2346 Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2347 fVal |= VMX_PIN_CTLS_PREEMPT_TIMER;
2348 }
2349
2350#if 0
2351 /* Enable posted-interrupt processing. */
2352 if (pVM->hm.s.fPostedIntrs)
2353 {
2354 Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2355 Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2356 fVal |= VMX_PIN_CTL_POSTED_INT;
2357 }
2358#endif
2359
2360 if ((fVal & fZap) != fVal)
2361 {
2362 LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2363 pVM->hm.s.vmx.Msrs.PinCtls.n.disallowed0, fVal, fZap));
2364 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2365 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2366 }
2367
2368 /* Commit it to the VMCS and update our cache. */
2369 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2370 AssertRCReturn(rc, rc);
2371 pVCpu->hm.s.vmx.u32PinCtls = fVal;
2372
2373 return VINF_SUCCESS;
2374}
2375
2376
2377/**
2378 * Sets up secondary processor-based VM-execution controls in the VMCS.
2379 *
2380 * @returns VBox status code.
2381 * @param pVCpu The cross context virtual CPU structure.
2382 *
2383 * @remarks We don't really care about optimizing vmwrites here as it's done only
2384 * once per VM and hence we don't care about VMCS-field cache comparisons.
2385 */
2386static int hmR0VmxSetupProcCtls2(PVMCPU pVCpu)
2387{
2388 PVM pVM = pVCpu->CTX_SUFF(pVM);
2389 uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2390 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2391
2392 /* WBINVD causes a VM-exit. */
2393 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2394 fVal |= VMX_PROC_CTLS2_WBINVD_EXIT;
2395
2396 /* Enable EPT (aka nested-paging). */
2397 if (pVM->hm.s.fNestedPaging)
2398 fVal |= VMX_PROC_CTLS2_EPT;
2399
2400 /*
2401 * Enable the INVPCID instruction if supported by the hardware and we expose
2402 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2403 */
2404 if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
2405 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2406 fVal |= VMX_PROC_CTLS2_INVPCID;
2407
2408 /* Enable VPID. */
2409 if (pVM->hm.s.vmx.fVpid)
2410 fVal |= VMX_PROC_CTLS2_VPID;
2411
2412 /* Enable Unrestricted guest execution. */
2413 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2414 fVal |= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
2415
2416#if 0
2417 if (pVM->hm.s.fVirtApicRegs)
2418 {
2419 /* Enable APIC-register virtualization. */
2420 Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
2421 fVal |= VMX_PROC_CTLS2_APIC_REG_VIRT;
2422
2423 /* Enable virtual-interrupt delivery. */
2424 Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
2425 fVal |= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
2426 }
2427#endif
2428
2429 /* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
2430 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2431 * done dynamically. */
2432 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
2433 {
2434 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2435 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2436 fVal |= VMX_PROC_CTLS2_VIRT_APIC_ACCESS; /* Virtualize APIC accesses. */
2437 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2438 AssertRCReturn(rc, rc);
2439 }
2440
2441 /* Enable RDTSCP. */
2442 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
2443 fVal |= VMX_PROC_CTLS2_RDTSCP;
2444
2445 /* Enable Pause-Loop exiting. */
2446 if ( pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT
2447 && pVM->hm.s.vmx.cPleGapTicks
2448 && pVM->hm.s.vmx.cPleWindowTicks)
2449 {
2450 fVal |= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
2451
2452 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2453 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2454 AssertRCReturn(rc, rc);
2455 }
2456
2457 if ((fVal & fZap) != fVal)
2458 {
2459 LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2460 pVM->hm.s.vmx.Msrs.ProcCtls2.n.disallowed0, fVal, fZap));
2461 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2462 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2463 }
2464
2465 /* Commit it to the VMCS and update our cache. */
2466 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2467 AssertRCReturn(rc, rc);
2468 pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2469
2470 return VINF_SUCCESS;
2471}
2472
2473
2474/**
2475 * Sets up processor-based VM-execution controls in the VMCS.
2476 *
2477 * @returns VBox status code.
2478 * @param pVCpu The cross context virtual CPU structure.
2479 *
2480 * @remarks We don't really care about optimizing vmwrites here as it's done only
2481 * once per VM and hence we don't care about VMCS-field cache comparisons.
2482 */
2483static int hmR0VmxSetupProcCtls(PVMCPU pVCpu)
2484{
2485 PVM pVM = pVCpu->CTX_SUFF(pVM);
2486 uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2487 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2488
2489 fVal |= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
2490 | VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2491 | VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2492 | VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2493 | VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2494 | VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2495 | VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2496
2497 /* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2498 if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
2499 || (pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
2500 {
2501 LogRelFunc(("Unsupported VMX_PROC_CTLS_MOV_DR_EXIT combo!"));
2502 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2503 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2504 }
2505
2506 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2507 if (!pVM->hm.s.fNestedPaging)
2508 {
2509 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2510 fVal |= VMX_PROC_CTLS_INVLPG_EXIT
2511 | VMX_PROC_CTLS_CR3_LOAD_EXIT
2512 | VMX_PROC_CTLS_CR3_STORE_EXIT;
2513 }
2514
2515 /* Use TPR shadowing if supported by the CPU. */
2516 if ( PDMHasApic(pVM)
2517 && pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
2518 {
2519 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2520 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2521 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2522 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2523 AssertRCReturn(rc, rc);
2524
2525 fVal |= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2526 /* CR8 writes cause a VM-exit based on TPR threshold. */
2527 Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
2528 Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
2529 }
2530 else
2531 {
2532 /*
2533 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2534 * Set this control only for 64-bit guests.
2535 */
2536 if (pVM->hm.s.fAllow64BitGuests)
2537 {
2538 fVal |= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2539 | VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2540 }
2541 }
2542
2543 /* Use MSR-bitmaps if supported by the CPU. */
2544 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2545 {
2546 fVal |= VMX_PROC_CTLS_USE_MSR_BITMAPS;
2547
2548 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2549 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2550 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2551 AssertRCReturn(rc, rc);
2552
2553 /*
2554 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2555 * automatically using dedicated fields in the VMCS.
2556 */
2557 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2558 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2559 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2560 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2561 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2562#if HC_ARCH_BITS == 64
2563 /*
2564 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2565 */
2566 if (pVM->hm.s.fAllow64BitGuests)
2567 {
2568 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2569 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2570 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2571 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2572 }
2573#endif
2574 /*
2575 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2576 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2577 */
2578 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2579 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2580
2581 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2582 }
2583
2584 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2585 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2586 fVal |= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
2587
2588 if ((fVal & fZap) != fVal)
2589 {
2590 LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2591 pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0, fVal, fZap));
2592 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2593 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2594 }
2595
2596 /* Commit it to the VMCS and update our cache. */
2597 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2598 AssertRCReturn(rc, rc);
2599 pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2600
2601 /* Set up secondary processor-based VM-execution controls if the CPU supports it. */
2602 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2603 return hmR0VmxSetupProcCtls2(pVCpu);
2604
2605 /* Sanity check, should not really happen. */
2606 if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2607 {
2608 LogRelFunc(("Unrestricted Guest enabled when secondary processor-based VM-execution controls not available\n"));
2609 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2610 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2611 }
2612
2613 /* Old CPUs without secondary processor-based VM-execution controls would end up here. */
2614 return VINF_SUCCESS;
2615}
2616
2617
2618/**
2619 * Sets up miscellaneous (everything other than Pin & Processor-based
2620 * VM-execution) control fields in the VMCS.
2621 *
2622 * @returns VBox status code.
2623 * @param pVCpu The cross context virtual CPU structure.
2624 */
2625static int hmR0VmxSetupMiscCtls(PVMCPU pVCpu)
2626{
2627 AssertPtr(pVCpu);
2628
2629 int rc = VERR_GENERAL_FAILURE;
2630
2631 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2632#if 0
2633 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2634 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2635 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2636
2637 /*
2638 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2639 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2640 * We thus use the exception bitmap to control it rather than use both.
2641 */
2642 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2643 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2644
2645 /* All IO & IOIO instructions cause VM-exits. */
2646 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2647 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2648
2649 /* Initialize the MSR-bitmap area. */
2650 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2651 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2652 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2653 AssertRCReturn(rc, rc);
2654#endif
2655
2656 /* Setup MSR auto-load/store area. */
2657 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2658 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2659 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2660 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2661 AssertRCReturn(rc, rc);
2662
2663 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2664 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2665 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2666 AssertRCReturn(rc, rc);
2667
2668 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2669 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2670 AssertRCReturn(rc, rc);
2671
2672 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2673#if 0
2674 /* Setup debug controls */
2675 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2676 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2677 AssertRCReturn(rc, rc);
2678#endif
2679
2680 return rc;
2681}
2682
2683
2684/**
2685 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2686 *
2687 * We shall setup those exception intercepts that don't change during the
2688 * lifetime of the VM here. The rest are done dynamically while loading the
2689 * guest state.
2690 *
2691 * @returns VBox status code.
2692 * @param pVCpu The cross context virtual CPU structure.
2693 */
2694static int hmR0VmxInitXcptBitmap(PVMCPU pVCpu)
2695{
2696 AssertPtr(pVCpu);
2697
2698 uint32_t uXcptBitmap;
2699
2700 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2701 uXcptBitmap = RT_BIT_32(X86_XCPT_AC);
2702
2703 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2704 and writes, and because recursive #DBs can cause the CPU hang, we must always
2705 intercept #DB. */
2706 uXcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2707
2708 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2709 if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
2710 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
2711
2712 /* Commit it to the VMCS. */
2713 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2714 AssertRCReturn(rc, rc);
2715
2716 /* Update our cache of the exception bitmap. */
2717 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
2718 return VINF_SUCCESS;
2719}
2720
2721
2722/**
2723 * Does per-VM VT-x initialization.
2724 *
2725 * @returns VBox status code.
2726 * @param pVM The cross context VM structure.
2727 */
2728VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2729{
2730 LogFlowFunc(("pVM=%p\n", pVM));
2731
2732 int rc = hmR0VmxStructsAlloc(pVM);
2733 if (RT_FAILURE(rc))
2734 {
2735 LogRelFunc(("hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2736 return rc;
2737 }
2738
2739 return VINF_SUCCESS;
2740}
2741
2742
2743/**
2744 * Does per-VM VT-x termination.
2745 *
2746 * @returns VBox status code.
2747 * @param pVM The cross context VM structure.
2748 */
2749VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2750{
2751 LogFlowFunc(("pVM=%p\n", pVM));
2752
2753#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2754 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2755 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2756#endif
2757 hmR0VmxStructsFree(pVM);
2758 return VINF_SUCCESS;
2759}
2760
2761
2762/**
2763 * Sets up the VM for execution under VT-x.
2764 * This function is only called once per-VM during initialization.
2765 *
2766 * @returns VBox status code.
2767 * @param pVM The cross context VM structure.
2768 */
2769VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2770{
2771 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2772 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2773
2774 LogFlowFunc(("pVM=%p\n", pVM));
2775
2776 /*
2777 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be
2778 * allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2779 * pRealModeTSS, see hmR3InitFinalizeR0Intel().
2780 */
2781 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2782 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2783 || !pVM->hm.s.vmx.pRealModeTSS))
2784 {
2785 LogRelFunc(("Invalid real-on-v86 state.\n"));
2786 return VERR_INTERNAL_ERROR;
2787 }
2788
2789 /* Initialize these always, see hmR3InitFinalizeR0().*/
2790 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
2791 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
2792
2793 /* Setup the tagged-TLB flush handlers. */
2794 int rc = hmR0VmxSetupTaggedTlb(pVM);
2795 if (RT_FAILURE(rc))
2796 {
2797 LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2798 return rc;
2799 }
2800
2801 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2802 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2803#if HC_ARCH_BITS == 64
2804 if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
2805 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2806 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
2807 {
2808 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2809 }
2810#endif
2811
2812 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2813 RTCCUINTREG const uHostCR4 = ASMGetCR4();
2814 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2815 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2816
2817 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2818 {
2819 PVMCPU pVCpu = &pVM->aCpus[i];
2820 AssertPtr(pVCpu);
2821 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2822
2823 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2824 Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2825
2826 /* Set revision dword at the beginning of the VMCS structure. */
2827 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
2828
2829 /* Set the VMCS launch state to "clear", see Intel spec. 31.6 "Preparation and launch a virtual machine". */
2830 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2831 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc\n", rc),
2832 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2833
2834 /* Load this VMCS as the current VMCS. */
2835 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2836 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc\n", rc),
2837 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2838
2839 rc = hmR0VmxSetupPinCtls(pVCpu);
2840 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc\n", rc),
2841 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2842
2843 rc = hmR0VmxSetupProcCtls(pVCpu);
2844 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc\n", rc),
2845 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2846
2847 rc = hmR0VmxSetupMiscCtls(pVCpu);
2848 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc\n", rc),
2849 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2850
2851 rc = hmR0VmxInitXcptBitmap(pVCpu);
2852 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc\n", rc),
2853 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2854
2855#if HC_ARCH_BITS == 32
2856 rc = hmR0VmxInitVmcsReadCache(pVCpu);
2857 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc\n", rc),
2858 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2859#endif
2860
2861 /* Sync any CPU internal VMCS data back into our VMCS in memory. */
2862 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2863 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc\n", rc),
2864 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2865
2866 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2867
2868 hmR0VmxUpdateErrorRecord(pVCpu, rc);
2869 }
2870
2871 return VINF_SUCCESS;
2872}
2873
2874
2875/**
2876 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2877 * the VMCS.
2878 *
2879 * @returns VBox status code.
2880 */
2881static int hmR0VmxExportHostControlRegs(void)
2882{
2883 RTCCUINTREG uReg = ASMGetCR0();
2884 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2885 AssertRCReturn(rc, rc);
2886
2887 uReg = ASMGetCR3();
2888 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2889 AssertRCReturn(rc, rc);
2890
2891 uReg = ASMGetCR4();
2892 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2893 AssertRCReturn(rc, rc);
2894 return rc;
2895}
2896
2897
2898/**
2899 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2900 * the host-state area in the VMCS.
2901 *
2902 * @returns VBox status code.
2903 * @param pVCpu The cross context virtual CPU structure.
2904 */
2905static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2906{
2907#if HC_ARCH_BITS == 64
2908/**
2909 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2910 * requirements. See hmR0VmxExportHostSegmentRegs().
2911 */
2912# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2913 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2914 { \
2915 bool fValidSelector = true; \
2916 if ((selValue) & X86_SEL_LDT) \
2917 { \
2918 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2919 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2920 } \
2921 if (fValidSelector) \
2922 { \
2923 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2924 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2925 } \
2926 (selValue) = 0; \
2927 }
2928
2929 /*
2930 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2931 * should -not- save the messed up state without restoring the original host-state,
2932 * see @bugref{7240}.
2933 *
2934 * This apparently can happen (most likely the FPU changes), deal with it rather than
2935 * asserting. Was observed booting Solaris 10u10 32-bit guest.
2936 */
2937 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2938 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2939 {
2940 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2941 pVCpu->idCpu));
2942 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2943 }
2944 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2945#else
2946 RT_NOREF(pVCpu);
2947#endif
2948
2949 /*
2950 * Host DS, ES, FS and GS segment registers.
2951 */
2952#if HC_ARCH_BITS == 64
2953 RTSEL uSelDS = ASMGetDS();
2954 RTSEL uSelES = ASMGetES();
2955 RTSEL uSelFS = ASMGetFS();
2956 RTSEL uSelGS = ASMGetGS();
2957#else
2958 RTSEL uSelDS = 0;
2959 RTSEL uSelES = 0;
2960 RTSEL uSelFS = 0;
2961 RTSEL uSelGS = 0;
2962#endif
2963
2964 /*
2965 * Host CS and SS segment registers.
2966 */
2967 RTSEL uSelCS = ASMGetCS();
2968 RTSEL uSelSS = ASMGetSS();
2969
2970 /*
2971 * Host TR segment register.
2972 */
2973 RTSEL uSelTR = ASMGetTR();
2974
2975#if HC_ARCH_BITS == 64
2976 /*
2977 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2978 * gain VM-entry and restore them before we get preempted.
2979 *
2980 * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2981 */
2982 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2983 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2984 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2985 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2986# undef VMXLOCAL_ADJUST_HOST_SEG
2987#endif
2988
2989 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2990 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2991 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2992 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2993 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2994 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2995 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2996 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2997 Assert(uSelCS);
2998 Assert(uSelTR);
2999
3000 /* Assertion is right but we would not have updated u32ExitCtls yet. */
3001#if 0
3002 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE))
3003 Assert(uSelSS != 0);
3004#endif
3005
3006 /* Write these host selector fields into the host-state area in the VMCS. */
3007 int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3008 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3009#if HC_ARCH_BITS == 64
3010 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3011 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3012 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3013 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3014#else
3015 NOREF(uSelDS);
3016 NOREF(uSelES);
3017 NOREF(uSelFS);
3018 NOREF(uSelGS);
3019#endif
3020 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3021 AssertRCReturn(rc, rc);
3022
3023 /*
3024 * Host GDTR and IDTR.
3025 */
3026 RTGDTR Gdtr;
3027 RTIDTR Idtr;
3028 RT_ZERO(Gdtr);
3029 RT_ZERO(Idtr);
3030 ASMGetGDTR(&Gdtr);
3031 ASMGetIDTR(&Idtr);
3032 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3033 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3034 AssertRCReturn(rc, rc);
3035
3036#if HC_ARCH_BITS == 64
3037 /*
3038 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
3039 * them to the maximum limit (0xffff) on every VM-exit.
3040 */
3041 if (Gdtr.cbGdt != 0xffff)
3042 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3043
3044 /*
3045 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
3046 * Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
3047 * as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
3048 * However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
3049 * uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
3050 * alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
3051 * at 0xffff on hosts where we are sure it won't cause trouble.
3052 */
3053# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3054 if (Idtr.cbIdt < 0x0fff)
3055# else
3056 if (Idtr.cbIdt != 0xffff)
3057# endif
3058 {
3059 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3060 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3061 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3062 }
3063#endif
3064
3065 /*
3066 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3067 * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3068 * RPL should be too in most cases.
3069 */
3070 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3071 ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3072
3073 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3074#if HC_ARCH_BITS == 64
3075 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3076
3077 /*
3078 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3079 * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3080 * restoration if the host has something else. Task switching is not supported in 64-bit
3081 * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3082 * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3083 *
3084 * [1] See Intel spec. 3.5 "System Descriptor Types".
3085 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3086 */
3087 PVM pVM = pVCpu->CTX_SUFF(pVM);
3088 Assert(pDesc->System.u4Type == 11);
3089 if ( pDesc->System.u16LimitLow != 0x67
3090 || pDesc->System.u4LimitHigh)
3091 {
3092 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3093 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3094 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3095 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3096 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3097 }
3098
3099 /*
3100 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3101 */
3102 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3103 {
3104 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3105 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3106 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3107 {
3108 /* The GDT is read-only but the writable GDT is available. */
3109 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3110 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3111 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3112 AssertRCReturn(rc, rc);
3113 }
3114 }
3115#else
3116 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3117#endif
3118 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3119 AssertRCReturn(rc, rc);
3120
3121 /*
3122 * Host FS base and GS base.
3123 */
3124#if HC_ARCH_BITS == 64
3125 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3126 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3127 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3128 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3129 AssertRCReturn(rc, rc);
3130
3131 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3132 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3133 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3134 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3135 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3136#endif
3137 return VINF_SUCCESS;
3138}
3139
3140
3141/**
3142 * Exports certain host MSRs in the VM-exit MSR-load area and some in the
3143 * host-state area of the VMCS.
3144 *
3145 * Theses MSRs will be automatically restored on the host after every successful
3146 * VM-exit.
3147 *
3148 * @returns VBox status code.
3149 * @param pVCpu The cross context virtual CPU structure.
3150 *
3151 * @remarks No-long-jump zone!!!
3152 */
3153static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3154{
3155 AssertPtr(pVCpu);
3156 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3157
3158 /*
3159 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3160 * rather than swapping them on every VM-entry.
3161 */
3162 hmR0VmxLazySaveHostMsrs(pVCpu);
3163
3164 /*
3165 * Host Sysenter MSRs.
3166 */
3167 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3168#if HC_ARCH_BITS == 32
3169 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3170 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3171#else
3172 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3173 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3174#endif
3175 AssertRCReturn(rc, rc);
3176
3177 /*
3178 * Host EFER MSR.
3179 *
3180 * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3181 * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3182 */
3183 PVM pVM = pVCpu->CTX_SUFF(pVM);
3184 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3185 {
3186 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3187 AssertRCReturn(rc, rc);
3188 }
3189
3190 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3191
3192 return VINF_SUCCESS;
3193}
3194
3195
3196/**
3197 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3198 *
3199 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3200 * these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3201 * hmR0VMxExportGuestEntryCtls().
3202 *
3203 * @returns true if we need to load guest EFER, false otherwise.
3204 * @param pVCpu The cross context virtual CPU structure.
3205 *
3206 * @remarks Requires EFER, CR4.
3207 * @remarks No-long-jump zone!!!
3208 */
3209static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu)
3210{
3211#ifdef HMVMX_ALWAYS_SWAP_EFER
3212 RT_NOREF(pVCpu);
3213 return true;
3214#else
3215
3216 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3217#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3218 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3219 if (CPUMIsGuestInLongModeEx(pCtx))
3220 return false;
3221#endif
3222
3223 PVM pVM = pVCpu->CTX_SUFF(pVM);
3224 uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3225 uint64_t const u64GuestEfer = pCtx->msrEFER;
3226
3227 /*
3228 * For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3229 * guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3230 */
3231 if ( CPUMIsGuestInLongModeEx(pCtx)
3232 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3233 {
3234 return true;
3235 }
3236
3237 /*
3238 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3239 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3240 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3241 */
3242 if ( (pCtx->cr4 & X86_CR4_PAE)
3243 && (pCtx->cr0 & X86_CR0_PG)
3244 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3245 {
3246 /* Assert that host is NX capable. */
3247 Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
3248 return true;
3249 }
3250
3251 return false;
3252#endif
3253}
3254
3255
3256/**
3257 * Exports the guest state with appropriate VM-entry controls in the VMCS.
3258 *
3259 * These controls can affect things done on VM-exit; e.g. "load debug controls",
3260 * see Intel spec. 24.8.1 "VM-entry controls".
3261 *
3262 * @returns VBox status code.
3263 * @param pVCpu The cross context virtual CPU structure.
3264 *
3265 * @remarks Requires EFER.
3266 * @remarks No-long-jump zone!!!
3267 */
3268static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu)
3269{
3270 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3271 {
3272 PVM pVM = pVCpu->CTX_SUFF(pVM);
3273 uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
3274 uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3275
3276 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3277 fVal |= VMX_ENTRY_CTLS_LOAD_DEBUG;
3278
3279 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3280 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
3281 {
3282 fVal |= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
3283 Log4Func(("VMX_ENTRY_CTLS_IA32E_MODE_GUEST\n"));
3284 }
3285 else
3286 Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
3287
3288 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3289 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3290 && hmR0VmxShouldSwapEferMsr(pVCpu))
3291 {
3292 fVal |= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
3293 Log4Func(("VMX_ENTRY_CTLS_LOAD_EFER_MSR\n"));
3294 }
3295
3296 /*
3297 * The following should -not- be set (since we're not in SMM mode):
3298 * - VMX_ENTRY_CTLS_ENTRY_TO_SMM
3299 * - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
3300 */
3301
3302 /** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
3303 * VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
3304
3305 if ((fVal & fZap) != fVal)
3306 {
3307 Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3308 pVM->hm.s.vmx.Msrs.EntryCtls.n.disallowed0, fVal, fZap));
3309 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3310 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3311 }
3312
3313 /* Commit it to the VMCS and update our cache. */
3314 if (pVCpu->hm.s.vmx.u32EntryCtls != fVal)
3315 {
3316 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3317 AssertRCReturn(rc, rc);
3318 pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3319 }
3320
3321 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3322 }
3323 return VINF_SUCCESS;
3324}
3325
3326
3327/**
3328 * Exports the guest state with appropriate VM-exit controls in the VMCS.
3329 *
3330 * @returns VBox status code.
3331 * @param pVCpu The cross context virtual CPU structure.
3332 *
3333 * @remarks Requires EFER.
3334 */
3335static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu)
3336{
3337 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3338 {
3339 PVM pVM = pVCpu->CTX_SUFF(pVM);
3340 uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
3341 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3342
3343 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3344 fVal |= VMX_EXIT_CTLS_SAVE_DEBUG;
3345
3346 /*
3347 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3348 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3349 * hmR0VmxExportHostMsrs().
3350 */
3351#if HC_ARCH_BITS == 64
3352 fVal |= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
3353 Log4Func(("VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE\n"));
3354#else
3355 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3356 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3357 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3358 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3359 {
3360 /* The switcher returns to long mode, EFER is managed by the switcher. */
3361 fVal |= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
3362 Log4Func(("VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE\n"));
3363 }
3364 else
3365 Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
3366#endif
3367
3368 /* If the newer VMCS fields for managing EFER exists, use it. */
3369 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3370 && hmR0VmxShouldSwapEferMsr(pVCpu))
3371 {
3372 fVal |= VMX_EXIT_CTLS_SAVE_EFER_MSR
3373 | VMX_EXIT_CTLS_LOAD_EFER_MSR;
3374 Log4Func(("VMX_EXIT_CTLS_SAVE_EFER_MSR and VMX_EXIT_CTLS_LOAD_EFER_MSR\n"));
3375 }
3376
3377 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3378 Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
3379
3380 /** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
3381 * VMX_EXIT_CTLS_SAVE_PAT_MSR,
3382 * VMX_EXIT_CTLS_LOAD_PAT_MSR. */
3383
3384 /* Enable saving of the VMX preemption timer value on VM-exit. */
3385 if ( pVM->hm.s.vmx.fUsePreemptTimer
3386 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
3387 fVal |= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
3388
3389 if ((fVal & fZap) != fVal)
3390 {
3391 LogRelFunc(("Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3392 pVM->hm.s.vmx.Msrs.ExitCtls.n.disallowed0, fVal, fZap));
3393 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3394 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3395 }
3396
3397 /* Commit it to the VMCS and update our cache. */
3398 if (pVCpu->hm.s.vmx.u32ExitCtls != fVal)
3399 {
3400 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3401 AssertRCReturn(rc, rc);
3402 pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3403 }
3404
3405 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3406 }
3407 return VINF_SUCCESS;
3408}
3409
3410
3411/**
3412 * Sets the TPR threshold in the VMCS.
3413 *
3414 * @returns VBox status code.
3415 * @param pVCpu The cross context virtual CPU structure.
3416 * @param u32TprThreshold The TPR threshold (task-priority class only).
3417 */
3418DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3419{
3420 Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
3421 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3422 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3423}
3424
3425
3426/**
3427 * Exports the guest APIC TPR state into the VMCS.
3428 *
3429 * @returns VBox status code.
3430 * @param pVCpu The cross context virtual CPU structure.
3431 *
3432 * @remarks No-long-jump zone!!!
3433 */
3434static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3435{
3436 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3437 {
3438 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
3439
3440 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3441 && APICIsEnabled(pVCpu))
3442 {
3443 /*
3444 * Setup TPR shadowing.
3445 */
3446 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
3447 {
3448 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3449
3450 bool fPendingIntr = false;
3451 uint8_t u8Tpr = 0;
3452 uint8_t u8PendingIntr = 0;
3453 int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3454 AssertRCReturn(rc, rc);
3455
3456 /*
3457 * If there are interrupts pending but masked by the TPR, instruct VT-x to
3458 * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3459 * priority of the pending interrupt so we can deliver the interrupt. If there
3460 * are no interrupts pending, set threshold to 0 to not cause any
3461 * TPR-below-threshold VM-exits.
3462 */
3463 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3464 uint32_t u32TprThreshold = 0;
3465 if (fPendingIntr)
3466 {
3467 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3468 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3469 const uint8_t u8TprPriority = u8Tpr >> 4;
3470 if (u8PendingPriority <= u8TprPriority)
3471 u32TprThreshold = u8PendingPriority;
3472 }
3473
3474 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3475 AssertRCReturn(rc, rc);
3476 }
3477 }
3478 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3479 }
3480 return VINF_SUCCESS;
3481}
3482
3483
3484/**
3485 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3486 *
3487 * @returns Guest's interruptibility-state.
3488 * @param pVCpu The cross context virtual CPU structure.
3489 *
3490 * @remarks No-long-jump zone!!!
3491 */
3492static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu)
3493{
3494 /*
3495 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3496 */
3497 uint32_t fIntrState = 0;
3498 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3499 {
3500 /* If inhibition is active, RIP & RFLAGS should've been accessed
3501 (i.e. read previously from the VMCS or from ring-3). */
3502 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3503#ifdef VBOX_STRICT
3504 uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
3505 AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3506#endif
3507 if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3508 {
3509 if (pCtx->eflags.Bits.u1IF)
3510 fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
3511 else
3512 fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
3513 }
3514 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3515 {
3516 /*
3517 * We can clear the inhibit force flag as even if we go back to the recompiler
3518 * without executing guest code in VT-x, the flag's condition to be cleared is
3519 * met and thus the cleared state is correct.
3520 */
3521 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3522 }
3523 }
3524
3525 /*
3526 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3527 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3528 * setting this would block host-NMIs and IRET will not clear the blocking.
3529 *
3530 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3531 */
3532 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3533 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
3534 {
3535 fIntrState |= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
3536 }
3537
3538 return fIntrState;
3539}
3540
3541
3542/**
3543 * Exports the guest's interruptibility-state into the guest-state area in the
3544 * VMCS.
3545 *
3546 * @returns VBox status code.
3547 * @param pVCpu The cross context virtual CPU structure.
3548 * @param fIntrState The interruptibility-state to set.
3549 */
3550static int hmR0VmxExportGuestIntrState(PVMCPU pVCpu, uint32_t fIntrState)
3551{
3552 NOREF(pVCpu);
3553 AssertMsg(!(fIntrState & 0xfffffff0), ("%#x\n", fIntrState)); /* Bits 31:4 MBZ. */
3554 Assert((fIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3555 return VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
3556}
3557
3558
3559/**
3560 * Exports the exception intercepts required for guest execution in the VMCS.
3561 *
3562 * @returns VBox status code.
3563 * @param pVCpu The cross context virtual CPU structure.
3564 *
3565 * @remarks No-long-jump zone!!!
3566 */
3567static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3568{
3569 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3570 {
3571 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3572
3573 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3574 if (pVCpu->hm.s.fGIMTrapXcptUD)
3575 uXcptBitmap |= RT_BIT(X86_XCPT_UD);
3576#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3577 else
3578 uXcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3579#endif
3580
3581 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_AC));
3582 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_DB));
3583
3584 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3585 {
3586 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3587 AssertRCReturn(rc, rc);
3588 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3589 }
3590
3591 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3592 Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", uXcptBitmap));
3593 }
3594 return VINF_SUCCESS;
3595}
3596
3597
3598/**
3599 * Exports the guest's RIP into the guest-state area in the VMCS.
3600 *
3601 * @returns VBox status code.
3602 * @param pVCpu The cross context virtual CPU structure.
3603 *
3604 * @remarks No-long-jump zone!!!
3605 */
3606static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
3607{
3608 int rc = VINF_SUCCESS;
3609 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3610 {
3611 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
3612
3613 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
3614 AssertRCReturn(rc, rc);
3615
3616 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3617 Log4Func(("RIP=%#RX64\n", pVCpu->cpum.GstCtx.rip));
3618 }
3619 return rc;
3620}
3621
3622
3623/**
3624 * Exports the guest's RSP into the guest-state area in the VMCS.
3625 *
3626 * @returns VBox status code.
3627 * @param pVCpu The cross context virtual CPU structure.
3628 *
3629 * @remarks No-long-jump zone!!!
3630 */
3631static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
3632{
3633 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3634 {
3635 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
3636
3637 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
3638 AssertRCReturn(rc, rc);
3639
3640 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3641 }
3642 return VINF_SUCCESS;
3643}
3644
3645
3646/**
3647 * Exports the guest's RFLAGS into the guest-state area in the VMCS.
3648 *
3649 * @returns VBox status code.
3650 * @param pVCpu The cross context virtual CPU structure.
3651 *
3652 * @remarks No-long-jump zone!!!
3653 */
3654static int hmR0VmxExportGuestRflags(PVMCPU pVCpu)
3655{
3656 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3657 {
3658 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
3659
3660 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3661 Let us assert it as such and use 32-bit VMWRITE. */
3662 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
3663 X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
3664 Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3665 Assert(!(fEFlags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3666
3667 /*
3668 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3669 * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3670 * can run the real-mode guest code under Virtual 8086 mode.
3671 */
3672 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3673 {
3674 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3675 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3676 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3677 fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3678 fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3679 }
3680
3681 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3682 AssertRCReturn(rc, rc);
3683
3684 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3685 Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3686 }
3687 return VINF_SUCCESS;
3688}
3689
3690
3691/**
3692 * Exports the guest CR0 control register into the guest-state area in the VMCS.
3693 *
3694 * The guest FPU state is always pre-loaded hence we don't need to bother about
3695 * sharing FPU related CR0 bits between the guest and host.
3696 *
3697 * @returns VBox status code.
3698 * @param pVCpu The cross context virtual CPU structure.
3699 *
3700 * @remarks No-long-jump zone!!!
3701 */
3702static int hmR0VmxExportGuestCR0(PVMCPU pVCpu)
3703{
3704 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3705 {
3706 PVM pVM = pVCpu->CTX_SUFF(pVM);
3707 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
3708 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.cr0));
3709
3710 uint32_t const u32ShadowCr0 = pVCpu->cpum.GstCtx.cr0;
3711 uint32_t u32GuestCr0 = pVCpu->cpum.GstCtx.cr0;
3712
3713 /*
3714 * Setup VT-x's view of the guest CR0.
3715 * Minimize VM-exits due to CR3 changes when we have NestedPaging.
3716 */
3717 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3718 if (pVM->hm.s.fNestedPaging)
3719 {
3720 if (CPUMIsGuestPagingEnabled(pVCpu))
3721 {
3722 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3723 uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
3724 | VMX_PROC_CTLS_CR3_STORE_EXIT);
3725 }
3726 else
3727 {
3728 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3729 uProcCtls |= VMX_PROC_CTLS_CR3_LOAD_EXIT
3730 | VMX_PROC_CTLS_CR3_STORE_EXIT;
3731 }
3732
3733 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3734 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3735 uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
3736 }
3737 else
3738 {
3739 /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3740 u32GuestCr0 |= X86_CR0_WP;
3741 }
3742
3743 /*
3744 * Guest FPU bits.
3745 *
3746 * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3747 * using CR0.TS.
3748 *
3749 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3750 * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3751 */
3752 u32GuestCr0 |= X86_CR0_NE;
3753
3754 /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3755 bool const fInterceptMF = !(u32ShadowCr0 & X86_CR0_NE);
3756
3757 /*
3758 * Update exception intercepts.
3759 */
3760 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3761 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3762 {
3763 Assert(PDMVmmDevHeapIsEnabled(pVM));
3764 Assert(pVM->hm.s.vmx.pRealModeTSS);
3765 uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3766 }
3767 else
3768 {
3769 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3770 uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3771 if (fInterceptMF)
3772 uXcptBitmap |= RT_BIT(X86_XCPT_MF);
3773 }
3774
3775 /* Additional intercepts for debugging, define these yourself explicitly. */
3776#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3777 uXcptBitmap |= 0
3778 | RT_BIT(X86_XCPT_BP)
3779 | RT_BIT(X86_XCPT_DE)
3780 | RT_BIT(X86_XCPT_NM)
3781 | RT_BIT(X86_XCPT_TS)
3782 | RT_BIT(X86_XCPT_UD)
3783 | RT_BIT(X86_XCPT_NP)
3784 | RT_BIT(X86_XCPT_SS)
3785 | RT_BIT(X86_XCPT_GP)
3786 | RT_BIT(X86_XCPT_PF)
3787 | RT_BIT(X86_XCPT_MF)
3788 ;
3789#elif defined(HMVMX_ALWAYS_TRAP_PF)
3790 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
3791#endif
3792 Assert(pVM->hm.s.fNestedPaging || (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
3793
3794 /*
3795 * Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3796 */
3797 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3798 uint32_t fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3799 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3800 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
3801 else
3802 Assert((fSetCr0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3803
3804 u32GuestCr0 |= fSetCr0;
3805 u32GuestCr0 &= fZapCr0;
3806 u32GuestCr0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3807
3808 /*
3809 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3810 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3811 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3812 */
3813 uint32_t u32Cr0Mask = X86_CR0_PE
3814 | X86_CR0_NE
3815 | (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3816 | X86_CR0_PG
3817 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3818 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3819 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3820
3821 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3822 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3823 * and @bugref{6944}. */
3824#if 0
3825 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3826 u32Cr0Mask &= ~X86_CR0_PE;
3827#endif
3828 /*
3829 * Finally, update VMCS fields with the CR0 values and the exception bitmap.
3830 */
3831 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCr0);
3832 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32ShadowCr0);
3833 if (u32Cr0Mask != pVCpu->hm.s.vmx.u32Cr0Mask)
3834 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32Cr0Mask);
3835 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3836 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3837 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3838 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3839 AssertRCReturn(rc, rc);
3840
3841 /* Update our caches. */
3842 pVCpu->hm.s.vmx.u32Cr0Mask = u32Cr0Mask;
3843 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3844 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3845
3846 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3847
3848 Log4Func(("u32Cr0Mask=%#RX32 u32ShadowCr0=%#RX32 u32GuestCr0=%#RX32 (fSetCr0=%#RX32 fZapCr0=%#RX32\n", u32Cr0Mask,
3849 u32ShadowCr0, u32GuestCr0, fSetCr0, fZapCr0));
3850 }
3851
3852 return VINF_SUCCESS;
3853}
3854
3855
3856/**
3857 * Exports the guest control registers (CR3, CR4) into the guest-state area
3858 * in the VMCS.
3859 *
3860 * @returns VBox strict status code.
3861 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3862 * without unrestricted guest access and the VMMDev is not presently
3863 * mapped (e.g. EFI32).
3864 *
3865 * @param pVCpu The cross context virtual CPU structure.
3866 *
3867 * @remarks No-long-jump zone!!!
3868 */
3869static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu)
3870{
3871 int rc = VINF_SUCCESS;
3872 PVM pVM = pVCpu->CTX_SUFF(pVM);
3873
3874 /*
3875 * Guest CR2.
3876 * It's always loaded in the assembler code. Nothing to do here.
3877 */
3878
3879 /*
3880 * Guest CR3.
3881 */
3882 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3883 {
3884 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
3885
3886 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3887 if (pVM->hm.s.fNestedPaging)
3888 {
3889 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3890
3891 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3892 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3893 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3894 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3895
3896 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3897 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3898 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3899
3900 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3901 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3902 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3903 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3904 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3905 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3906 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3907
3908 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3909 AssertRCReturn(rc, rc);
3910
3911 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3912 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3913 || CPUMIsGuestPagingEnabledEx(pCtx))
3914 {
3915 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3916 if (CPUMIsGuestInPAEModeEx(pCtx))
3917 {
3918 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3919 AssertRCReturn(rc, rc);
3920 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3921 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3922 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3923 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3924 AssertRCReturn(rc, rc);
3925 }
3926
3927 /*
3928 * The guest's view of its CR3 is unblemished with Nested Paging when the
3929 * guest is using paging or we have unrestricted guest execution to handle
3930 * the guest when it's not using paging.
3931 */
3932 GCPhysGuestCR3 = pCtx->cr3;
3933 }
3934 else
3935 {
3936 /*
3937 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
3938 * thinks it accesses physical memory directly, we use our identity-mapped
3939 * page table to map guest-linear to guest-physical addresses. EPT takes care
3940 * of translating it to host-physical addresses.
3941 */
3942 RTGCPHYS GCPhys;
3943 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3944
3945 /* We obtain it here every time as the guest could have relocated this PCI region. */
3946 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3947 if (RT_SUCCESS(rc))
3948 { /* likely */ }
3949 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3950 {
3951 Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3952 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3953 }
3954 else
3955 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3956
3957 GCPhysGuestCR3 = GCPhys;
3958 }
3959
3960 Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
3961 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3962 AssertRCReturn(rc, rc);
3963 }
3964 else
3965 {
3966 /* Non-nested paging case, just use the hypervisor's CR3. */
3967 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3968
3969 Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
3970 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3971 AssertRCReturn(rc, rc);
3972 }
3973
3974 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3975 }
3976
3977 /*
3978 * Guest CR4.
3979 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3980 */
3981 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3982 {
3983 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3984 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
3985 Assert(!RT_HI_U32(pCtx->cr4));
3986
3987 uint32_t u32GuestCr4 = pCtx->cr4;
3988 uint32_t const u32ShadowCr4 = pCtx->cr4;
3989
3990 /*
3991 * Setup VT-x's view of the guest CR4.
3992 *
3993 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3994 * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3995 * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3996 *
3997 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3998 */
3999 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4000 {
4001 Assert(pVM->hm.s.vmx.pRealModeTSS);
4002 Assert(PDMVmmDevHeapIsEnabled(pVM));
4003 u32GuestCr4 &= ~X86_CR4_VME;
4004 }
4005
4006 if (pVM->hm.s.fNestedPaging)
4007 {
4008 if ( !CPUMIsGuestPagingEnabledEx(pCtx)
4009 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4010 {
4011 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4012 u32GuestCr4 |= X86_CR4_PSE;
4013 /* Our identity mapping is a 32-bit page directory. */
4014 u32GuestCr4 &= ~X86_CR4_PAE;
4015 }
4016 /* else use guest CR4.*/
4017 }
4018 else
4019 {
4020 /*
4021 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4022 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4023 */
4024 switch (pVCpu->hm.s.enmShadowMode)
4025 {
4026 case PGMMODE_REAL: /* Real-mode. */
4027 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4028 case PGMMODE_32_BIT: /* 32-bit paging. */
4029 {
4030 u32GuestCr4 &= ~X86_CR4_PAE;
4031 break;
4032 }
4033
4034 case PGMMODE_PAE: /* PAE paging. */
4035 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4036 {
4037 u32GuestCr4 |= X86_CR4_PAE;
4038 break;
4039 }
4040
4041 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4042 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4043#ifdef VBOX_ENABLE_64_BITS_GUESTS
4044 break;
4045#endif
4046 default:
4047 AssertFailed();
4048 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4049 }
4050 }
4051
4052 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4053 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4054 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4055 u32GuestCr4 |= fSetCr4;
4056 u32GuestCr4 &= fZapCr4;
4057
4058 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them,
4059 that would cause a VM-exit. */
4060 uint32_t u32Cr4Mask = X86_CR4_VME
4061 | X86_CR4_PAE
4062 | X86_CR4_PGE
4063 | X86_CR4_PSE
4064 | X86_CR4_VMXE;
4065 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4066 u32Cr4Mask |= X86_CR4_OSXSAVE;
4067 if (pVM->cpum.ro.GuestFeatures.fPcid)
4068 u32Cr4Mask |= X86_CR4_PCIDE;
4069
4070 /* Write VT-x's view of the guest CR4, the CR4 modify mask and the read-only CR4 shadow
4071 into the VMCS and update our cache. */
4072 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCr4);
4073 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32ShadowCr4);
4074 if (pVCpu->hm.s.vmx.u32Cr4Mask != u32Cr4Mask)
4075 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32Cr4Mask);
4076 AssertRCReturn(rc, rc);
4077 pVCpu->hm.s.vmx.u32Cr4Mask = u32Cr4Mask;
4078
4079 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4080 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
4081
4082 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4083
4084 Log4Func(("u32GuestCr4=%#RX32 u32ShadowCr4=%#RX32 (fSetCr4=%#RX32 fZapCr4=%#RX32)\n", u32GuestCr4, u32ShadowCr4, fSetCr4,
4085 fZapCr4));
4086 }
4087 return rc;
4088}
4089
4090
4091/**
4092 * Exports the guest debug registers into the guest-state area in the VMCS.
4093 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4094 *
4095 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4096 *
4097 * @returns VBox status code.
4098 * @param pVCpu The cross context virtual CPU structure.
4099 *
4100 * @remarks No-long-jump zone!!!
4101 */
4102static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu)
4103{
4104 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4105
4106#ifdef VBOX_STRICT
4107 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4108 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
4109 {
4110 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4111 Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0);
4112 Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
4113 }
4114#endif
4115
4116 bool fSteppingDB = false;
4117 bool fInterceptMovDRx = false;
4118 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4119 if (pVCpu->hm.s.fSingleInstruction)
4120 {
4121 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4122 PVM pVM = pVCpu->CTX_SUFF(pVM);
4123 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
4124 {
4125 uProcCtls |= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
4126 Assert(fSteppingDB == false);
4127 }
4128 else
4129 {
4130 pVCpu->cpum.GstCtx.eflags.u32 |= X86_EFL_TF;
4131 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
4132 pVCpu->hm.s.fClearTrapFlag = true;
4133 fSteppingDB = true;
4134 }
4135 }
4136
4137 uint32_t u32GuestDr7;
4138 if ( fSteppingDB
4139 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4140 {
4141 /*
4142 * Use the combined guest and host DRx values found in the hypervisor register set
4143 * because the debugger has breakpoints active or someone is single stepping on the
4144 * host side without a monitor trap flag.
4145 *
4146 * Note! DBGF expects a clean DR6 state before executing guest code.
4147 */
4148#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4149 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4150 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4151 {
4152 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4153 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4154 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4155 }
4156 else
4157#endif
4158 if (!CPUMIsHyperDebugStateActive(pVCpu))
4159 {
4160 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4161 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4162 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4163 }
4164
4165 /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4166 u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4167 pVCpu->hm.s.fUsingHyperDR7 = true;
4168 fInterceptMovDRx = true;
4169 }
4170 else
4171 {
4172 /*
4173 * If the guest has enabled debug registers, we need to load them prior to
4174 * executing guest code so they'll trigger at the right time.
4175 */
4176 if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD))
4177 {
4178#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4179 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4180 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4181 {
4182 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4183 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4184 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4185 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4186 }
4187 else
4188#endif
4189 if (!CPUMIsGuestDebugStateActive(pVCpu))
4190 {
4191 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4192 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4193 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4194 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4195 }
4196 Assert(!fInterceptMovDRx);
4197 }
4198 /*
4199 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4200 * must intercept #DB in order to maintain a correct DR6 guest value, and
4201 * because we need to intercept it to prevent nested #DBs from hanging the
4202 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4203 */
4204#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4205 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4206 && !CPUMIsGuestDebugStateActive(pVCpu))
4207#else
4208 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4209#endif
4210 {
4211 fInterceptMovDRx = true;
4212 }
4213
4214 /* Update DR7 with the actual guest value. */
4215 u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
4216 pVCpu->hm.s.fUsingHyperDR7 = false;
4217 }
4218
4219 if (fInterceptMovDRx)
4220 uProcCtls |= VMX_PROC_CTLS_MOV_DR_EXIT;
4221 else
4222 uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
4223
4224 /*
4225 * Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
4226 * monitor-trap flag and update our cache.
4227 */
4228 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4229 {
4230 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4231 AssertRCReturn(rc2, rc2);
4232 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4233 }
4234
4235 /*
4236 * Update guest DR7.
4237 */
4238 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
4239 AssertRCReturn(rc, rc);
4240
4241 return VINF_SUCCESS;
4242}
4243
4244
4245#ifdef VBOX_STRICT
4246/**
4247 * Strict function to validate segment registers.
4248 *
4249 * @param pVCpu The cross context virtual CPU structure.
4250 *
4251 * @remarks Will import guest CR0 on strict builds during validation of
4252 * segments.
4253 */
4254static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu)
4255{
4256 /*
4257 * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4258 *
4259 * The reason we check for attribute value 0 in this function and not just the unusable bit is
4260 * because hmR0VmxExportGuestSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4261 * and doesn't change the guest-context value.
4262 */
4263 PVM pVM = pVCpu->CTX_SUFF(pVM);
4264 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4265 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4266 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4267 && ( !CPUMIsGuestInRealModeEx(pCtx)
4268 && !CPUMIsGuestInV86ModeEx(pCtx)))
4269 {
4270 /* Protected mode checks */
4271 /* CS */
4272 Assert(pCtx->cs.Attr.n.u1Present);
4273 Assert(!(pCtx->cs.Attr.u & 0xf00));
4274 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4275 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4276 || !(pCtx->cs.Attr.n.u1Granularity));
4277 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4278 || (pCtx->cs.Attr.n.u1Granularity));
4279 /* CS cannot be loaded with NULL in protected mode. */
4280 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4281 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4282 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4283 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4284 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4285 else
4286 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4287 /* SS */
4288 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4289 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4290 if ( !(pCtx->cr0 & X86_CR0_PE)
4291 || pCtx->cs.Attr.n.u4Type == 3)
4292 {
4293 Assert(!pCtx->ss.Attr.n.u2Dpl);
4294 }
4295 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4296 {
4297 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4298 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4299 Assert(pCtx->ss.Attr.n.u1Present);
4300 Assert(!(pCtx->ss.Attr.u & 0xf00));
4301 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4302 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4303 || !(pCtx->ss.Attr.n.u1Granularity));
4304 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4305 || (pCtx->ss.Attr.n.u1Granularity));
4306 }
4307 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmentReg(). */
4308 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4309 {
4310 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4311 Assert(pCtx->ds.Attr.n.u1Present);
4312 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4313 Assert(!(pCtx->ds.Attr.u & 0xf00));
4314 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4315 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4316 || !(pCtx->ds.Attr.n.u1Granularity));
4317 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4318 || (pCtx->ds.Attr.n.u1Granularity));
4319 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4320 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4321 }
4322 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4323 {
4324 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4325 Assert(pCtx->es.Attr.n.u1Present);
4326 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4327 Assert(!(pCtx->es.Attr.u & 0xf00));
4328 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4329 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4330 || !(pCtx->es.Attr.n.u1Granularity));
4331 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4332 || (pCtx->es.Attr.n.u1Granularity));
4333 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4334 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4335 }
4336 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4337 {
4338 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4339 Assert(pCtx->fs.Attr.n.u1Present);
4340 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4341 Assert(!(pCtx->fs.Attr.u & 0xf00));
4342 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4343 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4344 || !(pCtx->fs.Attr.n.u1Granularity));
4345 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4346 || (pCtx->fs.Attr.n.u1Granularity));
4347 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4348 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4349 }
4350 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4351 {
4352 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4353 Assert(pCtx->gs.Attr.n.u1Present);
4354 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4355 Assert(!(pCtx->gs.Attr.u & 0xf00));
4356 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4357 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4358 || !(pCtx->gs.Attr.n.u1Granularity));
4359 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4360 || (pCtx->gs.Attr.n.u1Granularity));
4361 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4362 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4363 }
4364 /* 64-bit capable CPUs. */
4365# if HC_ARCH_BITS == 64
4366 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4367 Assert(!pCtx->ss.Attr.u || !RT_HI_U32(pCtx->ss.u64Base));
4368 Assert(!pCtx->ds.Attr.u || !RT_HI_U32(pCtx->ds.u64Base));
4369 Assert(!pCtx->es.Attr.u || !RT_HI_U32(pCtx->es.u64Base));
4370# endif
4371 }
4372 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4373 || ( CPUMIsGuestInRealModeEx(pCtx)
4374 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4375 {
4376 /* Real and v86 mode checks. */
4377 /* hmR0VmxExportGuestSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4378 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4379 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4380 {
4381 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4382 }
4383 else
4384 {
4385 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4386 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4387 }
4388
4389 /* CS */
4390 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4391 Assert(pCtx->cs.u32Limit == 0xffff);
4392 Assert(u32CSAttr == 0xf3);
4393 /* SS */
4394 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4395 Assert(pCtx->ss.u32Limit == 0xffff);
4396 Assert(u32SSAttr == 0xf3);
4397 /* DS */
4398 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4399 Assert(pCtx->ds.u32Limit == 0xffff);
4400 Assert(u32DSAttr == 0xf3);
4401 /* ES */
4402 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4403 Assert(pCtx->es.u32Limit == 0xffff);
4404 Assert(u32ESAttr == 0xf3);
4405 /* FS */
4406 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4407 Assert(pCtx->fs.u32Limit == 0xffff);
4408 Assert(u32FSAttr == 0xf3);
4409 /* GS */
4410 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4411 Assert(pCtx->gs.u32Limit == 0xffff);
4412 Assert(u32GSAttr == 0xf3);
4413 /* 64-bit capable CPUs. */
4414# if HC_ARCH_BITS == 64
4415 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4416 Assert(!u32SSAttr || !RT_HI_U32(pCtx->ss.u64Base));
4417 Assert(!u32DSAttr || !RT_HI_U32(pCtx->ds.u64Base));
4418 Assert(!u32ESAttr || !RT_HI_U32(pCtx->es.u64Base));
4419# endif
4420 }
4421}
4422#endif /* VBOX_STRICT */
4423
4424
4425/**
4426 * Exports a guest segment register into the guest-state area in the VMCS.
4427 *
4428 * @returns VBox status code.
4429 * @param pVCpu The cross context virtual CPU structure.
4430 * @param idxSel Index of the selector in the VMCS.
4431 * @param idxLimit Index of the segment limit in the VMCS.
4432 * @param idxBase Index of the segment base in the VMCS.
4433 * @param idxAccess Index of the access rights of the segment in the VMCS.
4434 * @param pSelReg Pointer to the segment selector.
4435 *
4436 * @remarks No-long-jump zone!!!
4437 */
4438static int hmR0VmxExportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
4439 PCCPUMSELREG pSelReg)
4440{
4441 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4442 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4443 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4444 AssertRCReturn(rc, rc);
4445
4446 uint32_t u32Access = pSelReg->Attr.u;
4447 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4448 {
4449 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4450 u32Access = 0xf3;
4451 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4452 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4453 }
4454 else
4455 {
4456 /*
4457 * The way to differentiate between whether this is really a null selector or was just
4458 * a selector loaded with 0 in real-mode is using the segment attributes. A selector
4459 * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4460 * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4461 * NULL selectors loaded in protected-mode have their attribute as 0.
4462 */
4463 if (!u32Access)
4464 u32Access = X86DESCATTR_UNUSABLE;
4465 }
4466
4467 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4468 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4469 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4470
4471 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4472 AssertRCReturn(rc, rc);
4473 return rc;
4474}
4475
4476
4477/**
4478 * Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4479 * into the guest-state area in the VMCS.
4480 *
4481 * @returns VBox status code.
4482 * @param pVCpu The cross context virtual CPU structure.
4483 *
4484 * @remarks Will import guest CR0 on strict builds during validation of
4485 * segments.
4486 * @remarks No-long-jump zone!!!
4487 */
4488static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu)
4489{
4490 int rc = VERR_INTERNAL_ERROR_5;
4491 PVM pVM = pVCpu->CTX_SUFF(pVM);
4492 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4493
4494 /*
4495 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4496 */
4497 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4498 {
4499#ifdef VBOX_WITH_REM
4500 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4501 {
4502 Assert(pVM->hm.s.vmx.pRealModeTSS);
4503 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4504 if ( pVCpu->hm.s.vmx.fWasInRealMode
4505 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4506 {
4507 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4508 in real-mode (e.g. OpenBSD 4.0) */
4509 REMFlushTBs(pVM);
4510 Log4Func(("Switch to protected mode detected!\n"));
4511 pVCpu->hm.s.vmx.fWasInRealMode = false;
4512 }
4513 }
4514#endif
4515 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
4516 {
4517 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
4518 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4519 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pCtx->cs.Attr.u;
4520 rc = HMVMX_EXPORT_SREG(CS, &pCtx->cs);
4521 AssertRCReturn(rc, rc);
4522 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
4523 }
4524
4525 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
4526 {
4527 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
4528 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4529 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pCtx->ss.Attr.u;
4530 rc = HMVMX_EXPORT_SREG(SS, &pCtx->ss);
4531 AssertRCReturn(rc, rc);
4532 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
4533 }
4534
4535 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
4536 {
4537 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
4538 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4539 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pCtx->ds.Attr.u;
4540 rc = HMVMX_EXPORT_SREG(DS, &pCtx->ds);
4541 AssertRCReturn(rc, rc);
4542 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
4543 }
4544
4545 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
4546 {
4547 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
4548 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4549 pVCpu->hm.s.vmx.RealMode.AttrES.u = pCtx->es.Attr.u;
4550 rc = HMVMX_EXPORT_SREG(ES, &pCtx->es);
4551 AssertRCReturn(rc, rc);
4552 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
4553 }
4554
4555 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
4556 {
4557 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
4558 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4559 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pCtx->fs.Attr.u;
4560 rc = HMVMX_EXPORT_SREG(FS, &pCtx->fs);
4561 AssertRCReturn(rc, rc);
4562 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
4563 }
4564
4565 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
4566 {
4567 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
4568 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4569 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pCtx->gs.Attr.u;
4570 rc = HMVMX_EXPORT_SREG(GS, &pCtx->gs);
4571 AssertRCReturn(rc, rc);
4572 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
4573 }
4574
4575#ifdef VBOX_STRICT
4576 hmR0VmxValidateSegmentRegs(pVCpu);
4577#endif
4578
4579 Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pCtx->cs.Sel, pCtx->cs.u64Base,
4580 pCtx->cs.u32Limit, pCtx->cs.Attr.u));
4581 }
4582
4583 /*
4584 * Guest TR.
4585 */
4586 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4587 {
4588 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
4589
4590 /*
4591 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
4592 * achieved using the interrupt redirection bitmap (all bits cleared to let the guest
4593 * handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4594 */
4595 uint16_t u16Sel = 0;
4596 uint32_t u32Limit = 0;
4597 uint64_t u64Base = 0;
4598 uint32_t u32AccessRights = 0;
4599
4600 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4601 {
4602 u16Sel = pCtx->tr.Sel;
4603 u32Limit = pCtx->tr.u32Limit;
4604 u64Base = pCtx->tr.u64Base;
4605 u32AccessRights = pCtx->tr.Attr.u;
4606 }
4607 else
4608 {
4609 Assert(pVM->hm.s.vmx.pRealModeTSS);
4610 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
4611
4612 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4613 RTGCPHYS GCPhys;
4614 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4615 AssertRCReturn(rc, rc);
4616
4617 X86DESCATTR DescAttr;
4618 DescAttr.u = 0;
4619 DescAttr.n.u1Present = 1;
4620 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4621
4622 u16Sel = 0;
4623 u32Limit = HM_VTX_TSS_SIZE;
4624 u64Base = GCPhys; /* in real-mode phys = virt. */
4625 u32AccessRights = DescAttr.u;
4626 }
4627
4628 /* Validate. */
4629 Assert(!(u16Sel & RT_BIT(2)));
4630 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4631 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4632 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4633 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4634 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4635 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4636 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4637 Assert( (u32Limit & 0xfff) == 0xfff
4638 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4639 Assert( !(pCtx->tr.u32Limit & 0xfff00000)
4640 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4641
4642 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4643 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4644 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4645 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4646 AssertRCReturn(rc, rc);
4647
4648 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4649 Log4Func(("TR base=%#RX64\n", pCtx->tr.u64Base));
4650 }
4651
4652 /*
4653 * Guest GDTR.
4654 */
4655 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4656 {
4657 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
4658
4659 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
4660 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
4661 AssertRCReturn(rc, rc);
4662
4663 /* Validate. */
4664 Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4665
4666 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4667 Log4Func(("GDTR base=%#RX64\n", pCtx->gdtr.pGdt));
4668 }
4669
4670 /*
4671 * Guest LDTR.
4672 */
4673 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4674 {
4675 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
4676
4677 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4678 uint32_t u32Access = 0;
4679 if (!pCtx->ldtr.Attr.u)
4680 u32Access = X86DESCATTR_UNUSABLE;
4681 else
4682 u32Access = pCtx->ldtr.Attr.u;
4683
4684 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
4685 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
4686 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4687 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
4688 AssertRCReturn(rc, rc);
4689
4690 /* Validate. */
4691 if (!(u32Access & X86DESCATTR_UNUSABLE))
4692 {
4693 Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4694 Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4695 Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4696 Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4697 Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4698 Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4699 Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
4700 || !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4701 Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
4702 || pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4703 }
4704
4705 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4706 Log4Func(("LDTR base=%#RX64\n", pCtx->ldtr.u64Base));
4707 }
4708
4709 /*
4710 * Guest IDTR.
4711 */
4712 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4713 {
4714 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
4715
4716 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
4717 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
4718 AssertRCReturn(rc, rc);
4719
4720 /* Validate. */
4721 Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4722
4723 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4724 Log4Func(("IDTR base=%#RX64\n", pCtx->idtr.pIdt));
4725 }
4726
4727 return VINF_SUCCESS;
4728}
4729
4730
4731/**
4732 * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4733 * areas.
4734 *
4735 * These MSRs will automatically be loaded to the host CPU on every successful
4736 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4737 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4738 * -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4739 *
4740 * Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4741 *
4742 * @returns VBox status code.
4743 * @param pVCpu The cross context virtual CPU structure.
4744 *
4745 * @remarks No-long-jump zone!!!
4746 */
4747static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu)
4748{
4749 AssertPtr(pVCpu);
4750 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4751
4752 /*
4753 * MSRs that we use the auto-load/store MSR area in the VMCS.
4754 * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4755 */
4756 PVM pVM = pVCpu->CTX_SUFF(pVM);
4757 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4758 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4759 {
4760 if (pVM->hm.s.fAllow64BitGuests)
4761 {
4762#if HC_ARCH_BITS == 32
4763 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS | CPUMCTX_EXTRN_KERNEL_GS_BASE);
4764
4765 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pCtx->msrLSTAR, false, NULL);
4766 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pCtx->msrSTAR, false, NULL);
4767 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pCtx->msrSFMASK, false, NULL);
4768 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, false, NULL);
4769 AssertRCReturn(rc, rc);
4770# ifdef LOG_ENABLED
4771 PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4772 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4773 Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4774# endif
4775#endif
4776 }
4777 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4778 }
4779
4780 /*
4781 * Guest Sysenter MSRs.
4782 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4783 * VM-exits on WRMSRs for these MSRs.
4784 */
4785 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4786 {
4787 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
4788
4789 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4790 {
4791 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
4792 AssertRCReturn(rc, rc);
4793 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4794 }
4795
4796 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4797 {
4798 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
4799 AssertRCReturn(rc, rc);
4800 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4801 }
4802
4803 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4804 {
4805 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
4806 AssertRCReturn(rc, rc);
4807 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4808 }
4809 }
4810
4811 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4812 {
4813 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
4814
4815 if (hmR0VmxShouldSwapEferMsr(pVCpu))
4816 {
4817 /*
4818 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4819 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4820 */
4821 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4822 {
4823 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
4824 AssertRCReturn(rc,rc);
4825 Log4Func(("EFER=%#RX64\n", pCtx->msrEFER));
4826 }
4827 else
4828 {
4829 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pCtx->msrEFER, false /* fUpdateHostMsr */,
4830 NULL /* pfAddedAndUpdated */);
4831 AssertRCReturn(rc, rc);
4832
4833 /* We need to intercept reads too, see @bugref{7386#c16}. */
4834 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
4835 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4836 Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pCtx->msrEFER,
4837 pVCpu->hm.s.vmx.cMsrs));
4838 }
4839 }
4840 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4841 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4842 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4843 }
4844
4845 return VINF_SUCCESS;
4846}
4847
4848
4849#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4850/**
4851 * Check if guest state allows safe use of 32-bit switcher again.
4852 *
4853 * Segment bases and protected mode structures must be 32-bit addressable
4854 * because the 32-bit switcher will ignore high dword when writing these VMCS
4855 * fields. See @bugref{8432} for details.
4856 *
4857 * @returns true if safe, false if must continue to use the 64-bit switcher.
4858 * @param pCtx Pointer to the guest-CPU context.
4859 *
4860 * @remarks No-long-jump zone!!!
4861 */
4862static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
4863{
4864 if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4865 if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4866 if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4867 if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4868 if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4869 if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4870 if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4871 if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4872 if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4873 if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4874
4875 /* All good, bases are 32-bit. */
4876 return true;
4877}
4878#endif
4879
4880
4881/**
4882 * Selects up the appropriate function to run guest code.
4883 *
4884 * @returns VBox status code.
4885 * @param pVCpu The cross context virtual CPU structure.
4886 *
4887 * @remarks No-long-jump zone!!!
4888 */
4889static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu)
4890{
4891 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4892 if (CPUMIsGuestInLongModeEx(pCtx))
4893 {
4894#ifndef VBOX_ENABLE_64_BITS_GUESTS
4895 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4896#endif
4897 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4898#if HC_ARCH_BITS == 32
4899 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4900 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4901 {
4902#ifdef VBOX_STRICT
4903 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4904 {
4905 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4906 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4907 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4908 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4909 | HM_CHANGED_VMX_ENTRY_CTLS
4910 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4911 }
4912#endif
4913 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4914
4915 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4916 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4917 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4918 Log4Func(("Selected 64-bit switcher\n"));
4919 }
4920#else
4921 /* 64-bit host. */
4922 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4923#endif
4924 }
4925 else
4926 {
4927 /* Guest is not in long mode, use the 32-bit handler. */
4928#if HC_ARCH_BITS == 32
4929 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4930 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4931 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4932 {
4933# ifdef VBOX_STRICT
4934 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4935 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4936 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4937 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4938 | HM_CHANGED_VMX_ENTRY_CTLS
4939 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4940# endif
4941 }
4942# ifdef VBOX_ENABLE_64_BITS_GUESTS
4943 /*
4944 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4945 * design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4946 * switcher flag because now we know the guest is in a sane state where it's safe
4947 * to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4948 * the much faster 32-bit switcher again.
4949 */
4950 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4951 {
4952 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4953 Log4Func(("Selected 32-bit switcher\n"));
4954 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4955 }
4956 else
4957 {
4958 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4959 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4960 || hmR0VmxIs32BitSwitcherSafe(pCtx))
4961 {
4962 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4963 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4964 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4965 | HM_CHANGED_VMX_ENTRY_CTLS
4966 | HM_CHANGED_VMX_EXIT_CTLS
4967 | HM_CHANGED_HOST_CONTEXT);
4968 Log4Func(("Selected 32-bit switcher (safe)\n"));
4969 }
4970 }
4971# else
4972 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4973# endif
4974#else
4975 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4976#endif
4977 }
4978 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4979 return VINF_SUCCESS;
4980}
4981
4982
4983/**
4984 * Wrapper for running the guest code in VT-x.
4985 *
4986 * @returns VBox status code, no informational status codes.
4987 * @param pVCpu The cross context virtual CPU structure.
4988 *
4989 * @remarks No-long-jump zone!!!
4990 */
4991DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu)
4992{
4993 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4994 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4995 pCtx->fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
4996
4997 /*
4998 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4999 * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
5000 * callee-saved and thus the need for this XMM wrapper.
5001 *
5002 * See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
5003 */
5004 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
5005 /** @todo Add stats for resume vs launch. */
5006 PVM pVM = pVCpu->CTX_SUFF(pVM);
5007#ifdef VBOX_WITH_KERNEL_USING_XMM
5008 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
5009#else
5010 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
5011#endif
5012 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5013 return rc;
5014}
5015
5016
5017/**
5018 * Reports world-switch error and dumps some useful debug info.
5019 *
5020 * @param pVCpu The cross context virtual CPU structure.
5021 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5022 * @param pVmxTransient Pointer to the VMX transient structure (only
5023 * exitReason updated).
5024 */
5025static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
5026{
5027 Assert(pVCpu);
5028 Assert(pVmxTransient);
5029 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
5030
5031 Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
5032 switch (rcVMRun)
5033 {
5034 case VERR_VMX_INVALID_VMXON_PTR:
5035 AssertFailed();
5036 break;
5037 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5038 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5039 {
5040 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5041 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5042 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
5043 AssertRC(rc);
5044
5045 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5046 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5047 Cannot do it here as we may have been long preempted. */
5048
5049#ifdef VBOX_STRICT
5050 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5051 pVmxTransient->uExitReason));
5052 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
5053 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5054 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5055 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5056 else
5057 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5058 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5059 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5060
5061 /* VMX control bits. */
5062 uint32_t u32Val;
5063 uint64_t u64Val;
5064 RTHCUINTREG uHCReg;
5065 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5066 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5067 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5068 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5069 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
5070 {
5071 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5072 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5073 }
5074 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5075 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5076 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5077 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5078 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5079 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5080 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5081 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5082 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5083 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5084 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5085 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5086 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5087 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5088 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5089 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5090 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5091 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5092 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5093 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5094 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5095 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5096 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5097 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5098 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5099 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5100 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5101 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5102 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5103 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5104 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5105 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5106 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5107 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5108 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5109 {
5110 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5111 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5112 }
5113
5114 /* Guest bits. */
5115 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5116 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
5117 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5118 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
5119 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5120 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
5121 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
5122 {
5123 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5124 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5125 }
5126
5127 /* Host bits. */
5128 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5129 Log4(("Host CR0 %#RHr\n", uHCReg));
5130 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5131 Log4(("Host CR3 %#RHr\n", uHCReg));
5132 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5133 Log4(("Host CR4 %#RHr\n", uHCReg));
5134
5135 RTGDTR HostGdtr;
5136 PCX86DESCHC pDesc;
5137 ASMGetGDTR(&HostGdtr);
5138 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5139 Log4(("Host CS %#08x\n", u32Val));
5140 if (u32Val < HostGdtr.cbGdt)
5141 {
5142 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5143 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5144 }
5145
5146 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5147 Log4(("Host DS %#08x\n", u32Val));
5148 if (u32Val < HostGdtr.cbGdt)
5149 {
5150 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5151 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5152 }
5153
5154 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5155 Log4(("Host ES %#08x\n", u32Val));
5156 if (u32Val < HostGdtr.cbGdt)
5157 {
5158 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5159 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5160 }
5161
5162 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5163 Log4(("Host FS %#08x\n", u32Val));
5164 if (u32Val < HostGdtr.cbGdt)
5165 {
5166 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5167 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5168 }
5169
5170 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5171 Log4(("Host GS %#08x\n", u32Val));
5172 if (u32Val < HostGdtr.cbGdt)
5173 {
5174 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5175 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5176 }
5177
5178 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5179 Log4(("Host SS %#08x\n", u32Val));
5180 if (u32Val < HostGdtr.cbGdt)
5181 {
5182 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5183 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5184 }
5185
5186 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5187 Log4(("Host TR %#08x\n", u32Val));
5188 if (u32Val < HostGdtr.cbGdt)
5189 {
5190 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5191 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5192 }
5193
5194 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5195 Log4(("Host TR Base %#RHv\n", uHCReg));
5196 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5197 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5198 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5199 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5200 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5201 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5202 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5203 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5204 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5205 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5206 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5207 Log4(("Host RSP %#RHv\n", uHCReg));
5208 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5209 Log4(("Host RIP %#RHv\n", uHCReg));
5210# if HC_ARCH_BITS == 64
5211 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5212 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5213 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5214 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5215 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5216 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5217# endif
5218#endif /* VBOX_STRICT */
5219 break;
5220 }
5221
5222 default:
5223 /* Impossible */
5224 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5225 break;
5226 }
5227}
5228
5229
5230#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5231#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5232# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5233#endif
5234#ifdef VBOX_STRICT
5235static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5236{
5237 switch (idxField)
5238 {
5239 case VMX_VMCS_GUEST_RIP:
5240 case VMX_VMCS_GUEST_RSP:
5241 case VMX_VMCS_GUEST_SYSENTER_EIP:
5242 case VMX_VMCS_GUEST_SYSENTER_ESP:
5243 case VMX_VMCS_GUEST_GDTR_BASE:
5244 case VMX_VMCS_GUEST_IDTR_BASE:
5245 case VMX_VMCS_GUEST_CS_BASE:
5246 case VMX_VMCS_GUEST_DS_BASE:
5247 case VMX_VMCS_GUEST_ES_BASE:
5248 case VMX_VMCS_GUEST_FS_BASE:
5249 case VMX_VMCS_GUEST_GS_BASE:
5250 case VMX_VMCS_GUEST_SS_BASE:
5251 case VMX_VMCS_GUEST_LDTR_BASE:
5252 case VMX_VMCS_GUEST_TR_BASE:
5253 case VMX_VMCS_GUEST_CR3:
5254 return true;
5255 }
5256 return false;
5257}
5258
5259static bool hmR0VmxIsValidReadField(uint32_t idxField)
5260{
5261 switch (idxField)
5262 {
5263 /* Read-only fields. */
5264 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5265 return true;
5266 }
5267 /* Remaining readable fields should also be writable. */
5268 return hmR0VmxIsValidWriteField(idxField);
5269}
5270#endif /* VBOX_STRICT */
5271
5272
5273/**
5274 * Executes the specified handler in 64-bit mode.
5275 *
5276 * @returns VBox status code (no informational status codes).
5277 * @param pVCpu The cross context virtual CPU structure.
5278 * @param enmOp The operation to perform.
5279 * @param cParams Number of parameters.
5280 * @param paParam Array of 32-bit parameters.
5281 */
5282VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
5283{
5284 PVM pVM = pVCpu->CTX_SUFF(pVM);
5285 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5286 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5287 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5288 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5289
5290#ifdef VBOX_STRICT
5291 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5292 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5293
5294 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5295 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5296#endif
5297
5298 /* Disable interrupts. */
5299 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5300
5301#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5302 RTCPUID idHostCpu = RTMpCpuId();
5303 CPUMR0SetLApic(pVCpu, idHostCpu);
5304#endif
5305
5306 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5307 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5308
5309 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5310 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5311 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5312
5313 /* Leave VMX Root Mode. */
5314 VMXDisable();
5315
5316 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5317
5318 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5319 CPUMSetHyperEIP(pVCpu, enmOp);
5320 for (int i = (int)cParams - 1; i >= 0; i--)
5321 CPUMPushHyper(pVCpu, paParam[i]);
5322
5323 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5324
5325 /* Call the switcher. */
5326 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
5327 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5328
5329 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5330 /* Make sure the VMX instructions don't cause #UD faults. */
5331 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5332
5333 /* Re-enter VMX Root Mode */
5334 int rc2 = VMXEnable(HCPhysCpuPage);
5335 if (RT_FAILURE(rc2))
5336 {
5337 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5338 ASMSetFlags(fOldEFlags);
5339 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5340 return rc2;
5341 }
5342
5343 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5344 AssertRC(rc2);
5345 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5346 Assert(!(ASMGetFlags() & X86_EFL_IF));
5347 ASMSetFlags(fOldEFlags);
5348 return rc;
5349}
5350
5351
5352/**
5353 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5354 * supporting 64-bit guests.
5355 *
5356 * @returns VBox status code.
5357 * @param fResume Whether to VMLAUNCH or VMRESUME.
5358 * @param pCtx Pointer to the guest-CPU context.
5359 * @param pCache Pointer to the VMCS cache.
5360 * @param pVM The cross context VM structure.
5361 * @param pVCpu The cross context virtual CPU structure.
5362 */
5363DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5364{
5365 NOREF(fResume);
5366
5367 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5368 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5369
5370#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5371 pCache->uPos = 1;
5372 pCache->interPD = PGMGetInterPaeCR3(pVM);
5373 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5374#endif
5375
5376#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5377 pCache->TestIn.HCPhysCpuPage = 0;
5378 pCache->TestIn.HCPhysVmcs = 0;
5379 pCache->TestIn.pCache = 0;
5380 pCache->TestOut.HCPhysVmcs = 0;
5381 pCache->TestOut.pCache = 0;
5382 pCache->TestOut.pCtx = 0;
5383 pCache->TestOut.eflags = 0;
5384#else
5385 NOREF(pCache);
5386#endif
5387
5388 uint32_t aParam[10];
5389 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5390 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5391 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5392 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5393 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5394 aParam[5] = 0;
5395 aParam[6] = VM_RC_ADDR(pVM, pVM);
5396 aParam[7] = 0;
5397 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5398 aParam[9] = 0;
5399
5400#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5401 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5402 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5403#endif
5404 int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5405
5406#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5407 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5408 Assert(pCtx->dr[4] == 10);
5409 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5410#endif
5411
5412#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5413 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5414 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5415 pVCpu->hm.s.vmx.HCPhysVmcs));
5416 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5417 pCache->TestOut.HCPhysVmcs));
5418 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5419 pCache->TestOut.pCache));
5420 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5421 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5422 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5423 pCache->TestOut.pCtx));
5424 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5425#endif
5426 NOREF(pCtx);
5427 return rc;
5428}
5429
5430
5431/**
5432 * Initialize the VMCS-Read cache.
5433 *
5434 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5435 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5436 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5437 * (those that have a 32-bit FULL & HIGH part).
5438 *
5439 * @returns VBox status code.
5440 * @param pVCpu The cross context virtual CPU structure.
5441 */
5442static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
5443{
5444#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5445 do { \
5446 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5447 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5448 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5449 ++cReadFields; \
5450 } while (0)
5451
5452 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5453 uint32_t cReadFields = 0;
5454
5455 /*
5456 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5457 * and serve to indicate exceptions to the rules.
5458 */
5459
5460 /* Guest-natural selector base fields. */
5461#if 0
5462 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5463 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5464 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5465#endif
5466 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5467 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5468 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5469 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5470 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5471 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5472 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5473 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5474 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5475 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5476 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5477 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5478#if 0
5479 /* Unused natural width guest-state fields. */
5480 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5481 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5482#endif
5483 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5484 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5485
5486 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5487 these 64-bit fields (using "FULL" and "HIGH" fields). */
5488#if 0
5489 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5490 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5491 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5492 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5493 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5494 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5495 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5496 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5497 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5498#endif
5499
5500 /* Natural width guest-state fields. */
5501 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5502#if 0
5503 /* Currently unused field. */
5504 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5505#endif
5506
5507 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5508 {
5509 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5510 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5511 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5512 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5513 }
5514 else
5515 {
5516 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5517 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5518 }
5519
5520#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5521 return VINF_SUCCESS;
5522}
5523
5524
5525/**
5526 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5527 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5528 * darwin, running 64-bit guests).
5529 *
5530 * @returns VBox status code.
5531 * @param pVCpu The cross context virtual CPU structure.
5532 * @param idxField The VMCS field encoding.
5533 * @param u64Val 16, 32 or 64-bit value.
5534 */
5535VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5536{
5537 int rc;
5538 switch (idxField)
5539 {
5540 /*
5541 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5542 */
5543 /* 64-bit Control fields. */
5544 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5545 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5546 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5547 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5548 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5549 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5550 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5551 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5552 case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
5553 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5554 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5555 case VMX_VMCS64_CTRL_EPTP_FULL:
5556 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5557 /* 64-bit Guest-state fields. */
5558 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5559 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5560 case VMX_VMCS64_GUEST_PAT_FULL:
5561 case VMX_VMCS64_GUEST_EFER_FULL:
5562 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5563 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5564 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5565 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5566 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5567 /* 64-bit Host-state fields. */
5568 case VMX_VMCS64_HOST_PAT_FULL:
5569 case VMX_VMCS64_HOST_EFER_FULL:
5570 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5571 {
5572 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5573 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5574 break;
5575 }
5576
5577 /*
5578 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5579 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5580 */
5581 /* Natural-width Guest-state fields. */
5582 case VMX_VMCS_GUEST_CR3:
5583 case VMX_VMCS_GUEST_ES_BASE:
5584 case VMX_VMCS_GUEST_CS_BASE:
5585 case VMX_VMCS_GUEST_SS_BASE:
5586 case VMX_VMCS_GUEST_DS_BASE:
5587 case VMX_VMCS_GUEST_FS_BASE:
5588 case VMX_VMCS_GUEST_GS_BASE:
5589 case VMX_VMCS_GUEST_LDTR_BASE:
5590 case VMX_VMCS_GUEST_TR_BASE:
5591 case VMX_VMCS_GUEST_GDTR_BASE:
5592 case VMX_VMCS_GUEST_IDTR_BASE:
5593 case VMX_VMCS_GUEST_RSP:
5594 case VMX_VMCS_GUEST_RIP:
5595 case VMX_VMCS_GUEST_SYSENTER_ESP:
5596 case VMX_VMCS_GUEST_SYSENTER_EIP:
5597 {
5598 if (!(RT_HI_U32(u64Val)))
5599 {
5600 /* If this field is 64-bit, VT-x will zero out the top bits. */
5601 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5602 }
5603 else
5604 {
5605 /* Assert that only the 32->64 switcher case should ever come here. */
5606 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5607 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5608 }
5609 break;
5610 }
5611
5612 default:
5613 {
5614 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5615 rc = VERR_INVALID_PARAMETER;
5616 break;
5617 }
5618 }
5619 AssertRCReturn(rc, rc);
5620 return rc;
5621}
5622
5623
5624/**
5625 * Queue up a VMWRITE by using the VMCS write cache.
5626 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5627 *
5628 * @param pVCpu The cross context virtual CPU structure.
5629 * @param idxField The VMCS field encoding.
5630 * @param u64Val 16, 32 or 64-bit value.
5631 */
5632VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5633{
5634 AssertPtr(pVCpu);
5635 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5636
5637 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5638 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5639
5640 /* Make sure there are no duplicates. */
5641 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5642 {
5643 if (pCache->Write.aField[i] == idxField)
5644 {
5645 pCache->Write.aFieldVal[i] = u64Val;
5646 return VINF_SUCCESS;
5647 }
5648 }
5649
5650 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5651 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5652 pCache->Write.cValidEntries++;
5653 return VINF_SUCCESS;
5654}
5655#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5656
5657
5658/**
5659 * Sets up the usage of TSC-offsetting and updates the VMCS.
5660 *
5661 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5662 * VMX preemption timer.
5663 *
5664 * @returns VBox status code.
5665 * @param pVCpu The cross context virtual CPU structure.
5666 *
5667 * @remarks No-long-jump zone!!!
5668 */
5669static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5670{
5671 bool fOffsettedTsc;
5672 bool fParavirtTsc;
5673 PVM pVM = pVCpu->CTX_SUFF(pVM);
5674 uint64_t uTscOffset;
5675 if (pVM->hm.s.vmx.fUsePreemptTimer)
5676 {
5677 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
5678
5679 /* Make sure the returned values have sane upper and lower boundaries. */
5680 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5681 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5682 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5683 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5684
5685 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5686 int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
5687 AssertRC(rc);
5688 }
5689 else
5690 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
5691
5692 if (fParavirtTsc)
5693 {
5694 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5695 information before every VM-entry, hence disable it for performance sake. */
5696#if 0
5697 int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5698 AssertRC(rc);
5699#endif
5700 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5701 }
5702
5703 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
5704 if ( fOffsettedTsc
5705 && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5706 {
5707 if (pVCpu->hm.s.vmx.u64TscOffset != uTscOffset)
5708 {
5709 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
5710 AssertRC(rc);
5711 pVCpu->hm.s.vmx.u64TscOffset = uTscOffset;
5712 }
5713
5714 if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
5715 {
5716 uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
5717 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5718 AssertRC(rc);
5719 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5720 }
5721 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5722 }
5723 else
5724 {
5725 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5726 if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
5727 {
5728 uProcCtls |= VMX_PROC_CTLS_RDTSC_EXIT;
5729 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5730 AssertRC(rc);
5731 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5732 }
5733 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5734 }
5735}
5736
5737
5738/**
5739 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5740 * VM-exit interruption info type.
5741 *
5742 * @returns The IEM exception flags.
5743 * @param uVector The event vector.
5744 * @param uVmxVectorType The VMX event type.
5745 *
5746 * @remarks This function currently only constructs flags required for
5747 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5748 * and CR2 aspects of an exception are not included).
5749 */
5750static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5751{
5752 uint32_t fIemXcptFlags;
5753 switch (uVmxVectorType)
5754 {
5755 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5756 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5757 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5758 break;
5759
5760 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5761 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5762 break;
5763
5764 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5765 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5766 break;
5767
5768 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5769 {
5770 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5771 if (uVector == X86_XCPT_BP)
5772 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5773 else if (uVector == X86_XCPT_OF)
5774 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5775 else
5776 {
5777 fIemXcptFlags = 0;
5778 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5779 }
5780 break;
5781 }
5782
5783 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5784 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5785 break;
5786
5787 default:
5788 fIemXcptFlags = 0;
5789 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5790 break;
5791 }
5792 return fIemXcptFlags;
5793}
5794
5795
5796/**
5797 * Sets an event as a pending event to be injected into the guest.
5798 *
5799 * @param pVCpu The cross context virtual CPU structure.
5800 * @param u32IntInfo The VM-entry interruption-information field.
5801 * @param cbInstr The VM-entry instruction length in bytes (for software
5802 * interrupts, exceptions and privileged software
5803 * exceptions).
5804 * @param u32ErrCode The VM-entry exception error code.
5805 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5806 * page-fault.
5807 *
5808 * @remarks Statistics counter assumes this is a guest event being injected or
5809 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5810 * always incremented.
5811 */
5812DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5813 RTGCUINTPTR GCPtrFaultAddress)
5814{
5815 Assert(!pVCpu->hm.s.Event.fPending);
5816 pVCpu->hm.s.Event.fPending = true;
5817 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5818 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5819 pVCpu->hm.s.Event.cbInstr = cbInstr;
5820 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5821}
5822
5823
5824/**
5825 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5826 *
5827 * @param pVCpu The cross context virtual CPU structure.
5828 */
5829DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
5830{
5831 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
5832 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5833 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5834 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5835 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5836}
5837
5838
5839/**
5840 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
5841 *
5842 * @param pVCpu The cross context virtual CPU structure.
5843 */
5844DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
5845{
5846 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
5847 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5848 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
5849 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5850 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5851}
5852
5853
5854/**
5855 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
5856 *
5857 * @param pVCpu The cross context virtual CPU structure.
5858 */
5859DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
5860{
5861 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
5862 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5863 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
5864 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5865 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5866}
5867
5868
5869#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5870/**
5871 * Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
5872 *
5873 * @param pVCpu The cross context virtual CPU structure.
5874 * @param u32ErrCode The error code for the general-protection exception.
5875 */
5876DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
5877{
5878 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
5879 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5880 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5881 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5882 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrCode, 0 /* GCPtrFaultAddress */);
5883}
5884
5885
5886/**
5887 * Sets a stack (\#SS) exception as pending-for-injection into the VM.
5888 *
5889 * @param pVCpu The cross context virtual CPU structure.
5890 * @param u32ErrCode The error code for the stack exception.
5891 */
5892DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
5893{
5894 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
5895 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5896 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5897 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5898 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrCode, 0 /* GCPtrFaultAddress */);
5899}
5900
5901
5902/**
5903 * Decodes the memory operand of an instruction that caused a VM-exit.
5904 *
5905 * The VM-exit qualification field provides the displacement field for memory
5906 * operand instructions, if any.
5907 *
5908 * @returns Strict VBox status code (i.e. informational status codes too).
5909 * @retval VINF_SUCCESS if the operand was successfully decoded.
5910 * @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
5911 * operand.
5912 * @param pVCpu The cross context virtual CPU structure.
5913 * @param uExitInstrInfo The VM-exit instruction information field.
5914 * @param enmMemAccess The memory operand's access type (read or write).
5915 * @param GCPtrDisp The instruction displacement field, if any. For
5916 * RIP-relative addressing pass RIP + displacement here.
5917 * @param pGCPtrMem Where to store the effective destination memory address.
5918 */
5919static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
5920 PRTGCPTR pGCPtrMem)
5921{
5922 Assert(pGCPtrMem);
5923 Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
5924 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_EFER | CPUMCTX_EXTRN_CR0);
5925
5926 static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
5927 static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
5928 AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
5929
5930 VMXEXITINSTRINFO ExitInstrInfo;
5931 ExitInstrInfo.u = uExitInstrInfo;
5932 uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
5933 uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
5934 bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
5935 uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
5936 uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
5937 bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
5938 uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
5939 bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
5940 bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
5941
5942 /*
5943 * Validate instruction information.
5944 * This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
5945 */
5946 AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
5947 ("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
5948 AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
5949 ("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
5950 AssertLogRelMsgReturn(fIsMemOperand,
5951 ("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
5952
5953 /*
5954 * Compute the complete effective address.
5955 *
5956 * See AMD instruction spec. 1.4.2 "SIB Byte Format"
5957 * See AMD spec. 4.5.2 "Segment Registers".
5958 */
5959 RTGCPTR GCPtrMem = GCPtrDisp;
5960 if (fBaseRegValid)
5961 GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
5962 if (fIdxRegValid)
5963 GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
5964
5965 RTGCPTR const GCPtrOff = GCPtrMem;
5966 if ( !fIsLongMode
5967 || iSegReg >= X86_SREG_FS)
5968 GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
5969 GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
5970
5971 /*
5972 * Validate effective address.
5973 * See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
5974 */
5975 uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
5976 Assert(cbAccess > 0);
5977 if (fIsLongMode)
5978 {
5979 if (X86_IS_CANONICAL(GCPtrMem))
5980 {
5981 *pGCPtrMem = GCPtrMem;
5982 return VINF_SUCCESS;
5983 }
5984
5985 /** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
5986 * "Data Limit Checks in 64-bit Mode". */
5987 Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
5988 hmR0VmxSetPendingXcptGP(pVCpu, 0);
5989 return VINF_HM_PENDING_XCPT;
5990 }
5991
5992 /*
5993 * This is a watered down version of iemMemApplySegment().
5994 * Parts that are not applicable for VMX instructions like real-or-v8086 mode
5995 * and segment CPL/DPL checks are skipped.
5996 */
5997 RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
5998 RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
5999 PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
6000
6001 /* Check if the segment is present and usable. */
6002 if ( pSel->Attr.n.u1Present
6003 && !pSel->Attr.n.u1Unusable)
6004 {
6005 Assert(pSel->Attr.n.u1DescType);
6006 if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
6007 {
6008 /* Check permissions for the data segment. */
6009 if ( enmMemAccess == VMXMEMACCESS_WRITE
6010 && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
6011 {
6012 Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6013 hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
6014 return VINF_HM_PENDING_XCPT;
6015 }
6016
6017 /* Check limits if it's a normal data segment. */
6018 if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
6019 {
6020 if ( GCPtrFirst32 > pSel->u32Limit
6021 || GCPtrLast32 > pSel->u32Limit)
6022 {
6023 Log4Func(("Data segment limit exceeded."
6024 "iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6025 GCPtrLast32, pSel->u32Limit));
6026 if (iSegReg == X86_SREG_SS)
6027 hmR0VmxSetPendingXcptSS(pVCpu, 0);
6028 else
6029 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6030 return VINF_HM_PENDING_XCPT;
6031 }
6032 }
6033 else
6034 {
6035 /* Check limits if it's an expand-down data segment.
6036 Note! The upper boundary is defined by the B bit, not the G bit! */
6037 if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
6038 || GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
6039 {
6040 Log4Func(("Expand-down data segment limit exceeded."
6041 "iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6042 GCPtrLast32, pSel->u32Limit));
6043 if (iSegReg == X86_SREG_SS)
6044 hmR0VmxSetPendingXcptSS(pVCpu, 0);
6045 else
6046 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6047 return VINF_HM_PENDING_XCPT;
6048 }
6049 }
6050 }
6051 else
6052 {
6053 /* Check permissions for the code segment. */
6054 if ( enmMemAccess == VMXMEMACCESS_WRITE
6055 || ( enmMemAccess == VMXMEMACCESS_READ
6056 && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
6057 {
6058 Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
6059 Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
6060 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6061 return VINF_HM_PENDING_XCPT;
6062 }
6063
6064 /* Check limits for the code segment (normal/expand-down not applicable for code segments). */
6065 if ( GCPtrFirst32 > pSel->u32Limit
6066 || GCPtrLast32 > pSel->u32Limit)
6067 {
6068 Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
6069 GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
6070 if (iSegReg == X86_SREG_SS)
6071 hmR0VmxSetPendingXcptSS(pVCpu, 0);
6072 else
6073 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6074 return VINF_HM_PENDING_XCPT;
6075 }
6076 }
6077 }
6078 else
6079 {
6080 Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6081 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6082 return VINF_HM_PENDING_XCPT;
6083 }
6084
6085 *pGCPtrMem = GCPtrMem;
6086 return VINF_SUCCESS;
6087}
6088
6089
6090/**
6091 * Perform the relevant VMX instruction checks for VM-exits that occurred due to the
6092 * guest attempting to execute a VMX instruction.
6093 *
6094 * @returns Strict VBox status code (i.e. informational status codes too).
6095 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
6096 * @retval VINF_HM_PENDING_XCPT if an exception was raised.
6097 *
6098 * @param pVCpu The cross context virtual CPU structure.
6099 * @param uExitReason The VM-exit reason.
6100 *
6101 * @todo NstVmx: Document other error codes when VM-exit is implemented.
6102 * @remarks No-long-jump zone!!!
6103 */
6104static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
6105{
6106 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS
6107 | CPUMCTX_EXTRN_HWVIRT);
6108
6109 if ( CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx)
6110 || ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6111 && !CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx)))
6112 {
6113 Log4Func(("In real/v86-mode or long-mode outside 64-bit code segment -> #UD\n"));
6114 hmR0VmxSetPendingXcptUD(pVCpu);
6115 return VINF_HM_PENDING_XCPT;
6116 }
6117
6118 if (uExitReason == VMX_EXIT_VMXON)
6119 {
6120 /*
6121 * We check CR4.VMXE because it is required to be always set while in VMX operation
6122 * by physical CPUs and our CR4 read shadow is only consulted when executing specific
6123 * instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
6124 * otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
6125 */
6126 if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
6127 {
6128 Log4Func(("CR4.VMXE is not set -> #UD\n"));
6129 hmR0VmxSetPendingXcptUD(pVCpu);
6130 return VINF_HM_PENDING_XCPT;
6131 }
6132 }
6133 else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
6134 {
6135 /*
6136 * The guest has not entered VMX operation but attempted to execute a VMX instruction
6137 * (other than VMXON), we need to raise a #UD.
6138 */
6139 Log4Func(("Not in VMX root mode -> #UD\n"));
6140 hmR0VmxSetPendingXcptUD(pVCpu);
6141 return VINF_HM_PENDING_XCPT;
6142 }
6143
6144 if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
6145 {
6146 /*
6147 * The nested-guest attempted to execute a VMX instruction, cause a VM-exit and let
6148 * the guest hypervisor deal with it.
6149 */
6150 /** @todo NSTVMX: Trigger a VM-exit */
6151 }
6152
6153 /*
6154 * VMX instructions require CPL 0 except in VMX non-root mode where the VM-exit intercept
6155 * (above) takes preceedence over the CPL check.
6156 */
6157 if (CPUMGetGuestCPL(pVCpu) > 0)
6158 {
6159 Log4Func(("CPL > 0 -> #GP(0)\n"));
6160 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6161 return VINF_HM_PENDING_XCPT;
6162 }
6163
6164 return VINF_SUCCESS;
6165}
6166
6167#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
6168
6169
6170/**
6171 * Handle a condition that occurred while delivering an event through the guest
6172 * IDT.
6173 *
6174 * @returns Strict VBox status code (i.e. informational status codes too).
6175 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
6176 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
6177 * to continue execution of the guest which will delivery the \#DF.
6178 * @retval VINF_EM_RESET if we detected a triple-fault condition.
6179 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
6180 *
6181 * @param pVCpu The cross context virtual CPU structure.
6182 * @param pVmxTransient Pointer to the VMX transient structure.
6183 *
6184 * @remarks No-long-jump zone!!!
6185 */
6186static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6187{
6188 uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
6189
6190 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
6191 rc2 |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
6192 AssertRCReturn(rc2, rc2);
6193
6194 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
6195 if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6196 {
6197 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
6198 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
6199
6200 /*
6201 * If the event was a software interrupt (generated with INT n) or a software exception
6202 * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
6203 * can handle the VM-exit and continue guest execution which will re-execute the
6204 * instruction rather than re-injecting the exception, as that can cause premature
6205 * trips to ring-3 before injection and involve TRPM which currently has no way of
6206 * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
6207 * the problem).
6208 */
6209 IEMXCPTRAISE enmRaise;
6210 IEMXCPTRAISEINFO fRaiseInfo;
6211 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6212 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6213 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6214 {
6215 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
6216 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6217 }
6218 else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
6219 {
6220 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
6221 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
6222 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
6223 /** @todo Make AssertMsgReturn as just AssertMsg later. */
6224 AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
6225 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
6226 uExitVectorType), VERR_VMX_IPE_5);
6227
6228 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
6229
6230 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
6231 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
6232 {
6233 pVmxTransient->fVectoringPF = true;
6234 enmRaise = IEMXCPTRAISE_PREV_EVENT;
6235 }
6236 }
6237 else
6238 {
6239 /*
6240 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
6241 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
6242 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
6243 */
6244 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6245 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6246 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
6247 enmRaise = IEMXCPTRAISE_PREV_EVENT;
6248 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6249 }
6250
6251 /*
6252 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
6253 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
6254 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
6255 * subsequent VM-entry would fail.
6256 *
6257 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
6258 */
6259 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
6260 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6261 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
6262 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
6263 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
6264 {
6265 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6266 }
6267
6268 switch (enmRaise)
6269 {
6270 case IEMXCPTRAISE_CURRENT_XCPT:
6271 {
6272 Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
6273 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
6274 Assert(rcStrict == VINF_SUCCESS);
6275 break;
6276 }
6277
6278 case IEMXCPTRAISE_PREV_EVENT:
6279 {
6280 uint32_t u32ErrCode;
6281 if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
6282 {
6283 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6284 AssertRCReturn(rc2, rc2);
6285 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6286 }
6287 else
6288 u32ErrCode = 0;
6289
6290 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
6291 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6292 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6293 0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
6294
6295 Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
6296 pVCpu->hm.s.Event.u32ErrCode));
6297 Assert(rcStrict == VINF_SUCCESS);
6298 break;
6299 }
6300
6301 case IEMXCPTRAISE_REEXEC_INSTR:
6302 Assert(rcStrict == VINF_SUCCESS);
6303 break;
6304
6305 case IEMXCPTRAISE_DOUBLE_FAULT:
6306 {
6307 /*
6308 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
6309 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
6310 */
6311 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6312 {
6313 pVmxTransient->fVectoringDoublePF = true;
6314 Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
6315 pVCpu->cpum.GstCtx.cr2));
6316 rcStrict = VINF_SUCCESS;
6317 }
6318 else
6319 {
6320 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6321 hmR0VmxSetPendingXcptDF(pVCpu);
6322 Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
6323 uIdtVector, uExitVector));
6324 rcStrict = VINF_HM_DOUBLE_FAULT;
6325 }
6326 break;
6327 }
6328
6329 case IEMXCPTRAISE_TRIPLE_FAULT:
6330 {
6331 Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
6332 rcStrict = VINF_EM_RESET;
6333 break;
6334 }
6335
6336 case IEMXCPTRAISE_CPU_HANG:
6337 {
6338 Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
6339 rcStrict = VERR_EM_GUEST_CPU_HANG;
6340 break;
6341 }
6342
6343 default:
6344 {
6345 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6346 rcStrict = VERR_VMX_IPE_2;
6347 break;
6348 }
6349 }
6350 }
6351 else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6352 && VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6353 && uExitVector != X86_XCPT_DF
6354 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
6355 {
6356 /*
6357 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6358 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6359 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6360 */
6361 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6362 {
6363 Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
6364 VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6365 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6366 }
6367 }
6368
6369 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
6370 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
6371 return rcStrict;
6372}
6373
6374
6375/**
6376 * Imports a guest segment register from the current VMCS into
6377 * the guest-CPU context.
6378 *
6379 * @returns VBox status code.
6380 * @param pVCpu The cross context virtual CPU structure.
6381 * @param idxSel Index of the selector in the VMCS.
6382 * @param idxLimit Index of the segment limit in the VMCS.
6383 * @param idxBase Index of the segment base in the VMCS.
6384 * @param idxAccess Index of the access rights of the segment in the VMCS.
6385 * @param pSelReg Pointer to the segment selector.
6386 *
6387 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6388 * do not log!
6389 *
6390 * @remarks Never call this function directly!!! Use the
6391 * HMVMX_IMPORT_SREG() macro as that takes care
6392 * of whether to read from the VMCS cache or not.
6393 */
6394static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6395 PCPUMSELREG pSelReg)
6396{
6397 NOREF(pVCpu);
6398
6399 uint32_t u32Sel;
6400 uint32_t u32Limit;
6401 uint32_t u32Attr;
6402 uint64_t u64Base;
6403 int rc = VMXReadVmcs32(idxSel, &u32Sel);
6404 rc |= VMXReadVmcs32(idxLimit, &u32Limit);
6405 rc |= VMXReadVmcs32(idxAccess, &u32Attr);
6406 rc |= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
6407 AssertRCReturn(rc, rc);
6408
6409 pSelReg->Sel = (uint16_t)u32Sel;
6410 pSelReg->ValidSel = (uint16_t)u32Sel;
6411 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6412 pSelReg->u32Limit = u32Limit;
6413 pSelReg->u64Base = u64Base;
6414 pSelReg->Attr.u = u32Attr;
6415
6416 /*
6417 * If VT-x marks the segment as unusable, most other bits remain undefined:
6418 * - For CS the L, D and G bits have meaning.
6419 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6420 * - For the remaining data segments no bits are defined.
6421 *
6422 * The present bit and the unusable bit has been observed to be set at the
6423 * same time (the selector was supposed to be invalid as we started executing
6424 * a V8086 interrupt in ring-0).
6425 *
6426 * What should be important for the rest of the VBox code, is that the P bit is
6427 * cleared. Some of the other VBox code recognizes the unusable bit, but
6428 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6429 * safe side here, we'll strip off P and other bits we don't care about. If
6430 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6431 *
6432 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6433 */
6434 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6435 {
6436 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6437
6438 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6439 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6440 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6441#ifdef VBOX_STRICT
6442 VMMRZCallRing3Disable(pVCpu);
6443 Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
6444# ifdef DEBUG_bird
6445 AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
6446 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6447 idxSel, u32Sel, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6448# endif
6449 VMMRZCallRing3Enable(pVCpu);
6450#endif
6451 }
6452 return VINF_SUCCESS;
6453}
6454
6455
6456/**
6457 * Imports the guest RIP from the VMCS back into the guest-CPU context.
6458 *
6459 * @returns VBox status code.
6460 * @param pVCpu The cross context virtual CPU structure.
6461 *
6462 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6463 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6464 * instead!!!
6465 */
6466DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6467{
6468 uint64_t u64Val;
6469 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6470 if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6471 {
6472 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6473 if (RT_SUCCESS(rc))
6474 {
6475 pCtx->rip = u64Val;
6476 EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
6477 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6478 }
6479 return rc;
6480 }
6481 return VINF_SUCCESS;
6482}
6483
6484
6485/**
6486 * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6487 *
6488 * @returns VBox status code.
6489 * @param pVCpu The cross context virtual CPU structure.
6490 *
6491 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6492 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6493 * instead!!!
6494 */
6495DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6496{
6497 uint32_t u32Val;
6498 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6499 if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6500 {
6501 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6502 if (RT_SUCCESS(rc))
6503 {
6504 pCtx->eflags.u32 = u32Val;
6505
6506 /* Restore eflags for real-on-v86-mode hack. */
6507 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6508 {
6509 pCtx->eflags.Bits.u1VM = 0;
6510 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6511 }
6512 }
6513 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6514 return rc;
6515 }
6516 return VINF_SUCCESS;
6517}
6518
6519
6520/**
6521 * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6522 * context.
6523 *
6524 * @returns VBox status code.
6525 * @param pVCpu The cross context virtual CPU structure.
6526 *
6527 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6528 * do not log!
6529 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6530 * instead!!!
6531 */
6532DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6533{
6534 uint32_t u32Val;
6535 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6536 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
6537 if (RT_SUCCESS(rc))
6538 {
6539 /*
6540 * We additionally have a requirement to import RIP, RFLAGS depending on whether we
6541 * might need them in hmR0VmxEvaluatePendingEvent().
6542 */
6543 if (!u32Val)
6544 {
6545 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6546 {
6547 rc = hmR0VmxImportGuestRip(pVCpu);
6548 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6549 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6550 }
6551
6552 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6553 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6554 }
6555 else
6556 {
6557 rc = hmR0VmxImportGuestRip(pVCpu);
6558 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6559
6560 if (u32Val & ( VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
6561 | VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
6562 {
6563 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6564 }
6565 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6566 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6567
6568 if (u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
6569 {
6570 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6571 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6572 }
6573 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6574 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6575 }
6576 }
6577 return rc;
6578}
6579
6580
6581/**
6582 * Worker for VMXR0ImportStateOnDemand.
6583 *
6584 * @returns VBox status code.
6585 * @param pVCpu The cross context virtual CPU structure.
6586 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6587 */
6588static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6589{
6590#define VMXLOCAL_BREAK_RC(a_rc) \
6591 if (RT_FAILURE(a_rc)) \
6592 break
6593
6594 int rc = VINF_SUCCESS;
6595 PVM pVM = pVCpu->CTX_SUFF(pVM);
6596 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6597 uint64_t u64Val;
6598 uint32_t u32Val;
6599
6600 Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6601 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6602
6603 /*
6604 * We disable interrupts to make the updating of the state and in particular
6605 * the fExtrn modification atomic wrt to preemption hooks.
6606 */
6607 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6608
6609 fWhat &= pCtx->fExtrn;
6610 if (fWhat)
6611 {
6612 do
6613 {
6614 if (fWhat & CPUMCTX_EXTRN_RIP)
6615 {
6616 rc = hmR0VmxImportGuestRip(pVCpu);
6617 VMXLOCAL_BREAK_RC(rc);
6618 }
6619
6620 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6621 {
6622 rc = hmR0VmxImportGuestRFlags(pVCpu);
6623 VMXLOCAL_BREAK_RC(rc);
6624 }
6625
6626 if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6627 {
6628 rc = hmR0VmxImportGuestIntrState(pVCpu);
6629 VMXLOCAL_BREAK_RC(rc);
6630 }
6631
6632 if (fWhat & CPUMCTX_EXTRN_RSP)
6633 {
6634 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6635 VMXLOCAL_BREAK_RC(rc);
6636 pCtx->rsp = u64Val;
6637 }
6638
6639 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6640 {
6641 if (fWhat & CPUMCTX_EXTRN_CS)
6642 {
6643 rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6644 rc |= hmR0VmxImportGuestRip(pVCpu);
6645 VMXLOCAL_BREAK_RC(rc);
6646 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6647 pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6648 EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true);
6649 }
6650 if (fWhat & CPUMCTX_EXTRN_SS)
6651 {
6652 rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6653 VMXLOCAL_BREAK_RC(rc);
6654 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6655 pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6656 }
6657 if (fWhat & CPUMCTX_EXTRN_DS)
6658 {
6659 rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6660 VMXLOCAL_BREAK_RC(rc);
6661 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6662 pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6663 }
6664 if (fWhat & CPUMCTX_EXTRN_ES)
6665 {
6666 rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6667 VMXLOCAL_BREAK_RC(rc);
6668 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6669 pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6670 }
6671 if (fWhat & CPUMCTX_EXTRN_FS)
6672 {
6673 rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6674 VMXLOCAL_BREAK_RC(rc);
6675 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6676 pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6677 }
6678 if (fWhat & CPUMCTX_EXTRN_GS)
6679 {
6680 rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6681 VMXLOCAL_BREAK_RC(rc);
6682 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6683 pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6684 }
6685 }
6686
6687 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6688 {
6689 if (fWhat & CPUMCTX_EXTRN_LDTR)
6690 {
6691 rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6692 VMXLOCAL_BREAK_RC(rc);
6693 }
6694
6695 if (fWhat & CPUMCTX_EXTRN_GDTR)
6696 {
6697 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6698 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6699 VMXLOCAL_BREAK_RC(rc);
6700 pCtx->gdtr.pGdt = u64Val;
6701 pCtx->gdtr.cbGdt = u32Val;
6702 }
6703
6704 /* Guest IDTR. */
6705 if (fWhat & CPUMCTX_EXTRN_IDTR)
6706 {
6707 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6708 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6709 VMXLOCAL_BREAK_RC(rc);
6710 pCtx->idtr.pIdt = u64Val;
6711 pCtx->idtr.cbIdt = u32Val;
6712 }
6713
6714 /* Guest TR. */
6715 if (fWhat & CPUMCTX_EXTRN_TR)
6716 {
6717 /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6718 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6719 {
6720 rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6721 VMXLOCAL_BREAK_RC(rc);
6722 }
6723 }
6724 }
6725
6726 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6727 {
6728 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6729 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6730 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6731 pCtx->SysEnter.cs = u32Val;
6732 VMXLOCAL_BREAK_RC(rc);
6733 }
6734
6735#if HC_ARCH_BITS == 64
6736 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6737 {
6738 if ( pVM->hm.s.fAllow64BitGuests
6739 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6740 pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6741 }
6742
6743 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6744 {
6745 if ( pVM->hm.s.fAllow64BitGuests
6746 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6747 {
6748 pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6749 pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6750 pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6751 }
6752 }
6753#endif
6754
6755 if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
6756#if HC_ARCH_BITS == 32
6757 || (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS))
6758#endif
6759 )
6760 {
6761 PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6762 uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6763 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6764 {
6765 switch (pMsr->u32Msr)
6766 {
6767#if HC_ARCH_BITS == 32
6768 case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6769 case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6770 case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6771 case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6772#endif
6773 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6774 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6775 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit */ break;
6776 default:
6777 {
6778 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6779 ASMSetFlags(fEFlags);
6780 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6781 cMsrs));
6782 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6783 }
6784 }
6785 }
6786 }
6787
6788 if (fWhat & CPUMCTX_EXTRN_DR7)
6789 {
6790 if (!pVCpu->hm.s.fUsingHyperDR7)
6791 {
6792 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6793 rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6794 VMXLOCAL_BREAK_RC(rc);
6795 pCtx->dr[7] = u32Val;
6796 }
6797 }
6798
6799 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6800 {
6801 uint32_t u32Shadow;
6802 if (fWhat & CPUMCTX_EXTRN_CR0)
6803 {
6804 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6805 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6806 VMXLOCAL_BREAK_RC(rc);
6807 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr0Mask)
6808 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr0Mask);
6809 VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6810 CPUMSetGuestCR0(pVCpu, u32Val);
6811 VMMRZCallRing3Enable(pVCpu);
6812 }
6813
6814 if (fWhat & CPUMCTX_EXTRN_CR4)
6815 {
6816 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6817 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6818 VMXLOCAL_BREAK_RC(rc);
6819 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr4Mask)
6820 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr4Mask);
6821 CPUMSetGuestCR4(pVCpu, u32Val);
6822 }
6823
6824 if (fWhat & CPUMCTX_EXTRN_CR3)
6825 {
6826 /* CR0.PG bit changes are always intercepted, so it's up to date. */
6827 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6828 || ( pVM->hm.s.fNestedPaging
6829 && CPUMIsGuestPagingEnabledEx(pCtx)))
6830 {
6831 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6832 if (pCtx->cr3 != u64Val)
6833 {
6834 CPUMSetGuestCR3(pVCpu, u64Val);
6835 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6836 }
6837
6838 /* If the guest is in PAE mode, sync back the PDPE's into the guest state.
6839 Note: CR4.PAE, CR0.PG, EFER bit changes are always intercepted, so they're up to date. */
6840 if (CPUMIsGuestInPAEModeEx(pCtx))
6841 {
6842 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6843 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6844 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6845 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6846 VMXLOCAL_BREAK_RC(rc);
6847 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6848 }
6849 }
6850 }
6851 }
6852 } while (0);
6853
6854 if (RT_SUCCESS(rc))
6855 {
6856 /* Update fExtrn. */
6857 pCtx->fExtrn &= ~fWhat;
6858
6859 /* If everything has been imported, clear the HM keeper bit. */
6860 if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6861 {
6862 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6863 Assert(!pCtx->fExtrn);
6864 }
6865 }
6866 }
6867 else
6868 AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6869
6870 ASMSetFlags(fEFlags);
6871
6872 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6873
6874 /*
6875 * Honor any pending CR3 updates.
6876 *
6877 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6878 * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6879 * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6880 *
6881 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6882 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6883 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6884 * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6885 *
6886 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6887 */
6888 if (VMMRZCallRing3IsEnabled(pVCpu))
6889 {
6890 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6891 {
6892 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
6893 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6894 }
6895
6896 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6897 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6898
6899 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6900 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6901 }
6902
6903 return VINF_SUCCESS;
6904#undef VMXLOCAL_BREAK_RC
6905}
6906
6907
6908/**
6909 * Saves the guest state from the VMCS into the guest-CPU context.
6910 *
6911 * @returns VBox status code.
6912 * @param pVCpu The cross context virtual CPU structure.
6913 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6914 */
6915VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6916{
6917 return hmR0VmxImportGuestState(pVCpu, fWhat);
6918}
6919
6920
6921/**
6922 * Check per-VM and per-VCPU force flag actions that require us to go back to
6923 * ring-3 for one reason or another.
6924 *
6925 * @returns Strict VBox status code (i.e. informational status codes too)
6926 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6927 * ring-3.
6928 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6929 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6930 * interrupts)
6931 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6932 * all EMTs to be in ring-3.
6933 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6934 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6935 * to the EM loop.
6936 *
6937 * @param pVCpu The cross context virtual CPU structure.
6938 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6939 */
6940static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
6941{
6942 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6943
6944 /*
6945 * Anything pending? Should be more likely than not if we're doing a good job.
6946 */
6947 PVM pVM = pVCpu->CTX_SUFF(pVM);
6948 if ( !fStepping
6949 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6950 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6951 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6952 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6953 return VINF_SUCCESS;
6954
6955 /* Pending PGM C3 sync. */
6956 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6957 {
6958 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6959 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
6960 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
6961 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6962 if (rcStrict2 != VINF_SUCCESS)
6963 {
6964 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6965 Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6966 return rcStrict2;
6967 }
6968 }
6969
6970 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6971 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6972 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6973 {
6974 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6975 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6976 Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6977 return rc2;
6978 }
6979
6980 /* Pending VM request packets, such as hardware interrupts. */
6981 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6982 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6983 {
6984 Log4Func(("Pending VM request forcing us back to ring-3\n"));
6985 return VINF_EM_PENDING_REQUEST;
6986 }
6987
6988 /* Pending PGM pool flushes. */
6989 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6990 {
6991 Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6992 return VINF_PGM_POOL_FLUSH_PENDING;
6993 }
6994
6995 /* Pending DMA requests. */
6996 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6997 {
6998 Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6999 return VINF_EM_RAW_TO_R3;
7000 }
7001
7002 return VINF_SUCCESS;
7003}
7004
7005
7006/**
7007 * Converts any TRPM trap into a pending HM event. This is typically used when
7008 * entering from ring-3 (not longjmp returns).
7009 *
7010 * @param pVCpu The cross context virtual CPU structure.
7011 */
7012static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7013{
7014 Assert(TRPMHasTrap(pVCpu));
7015 Assert(!pVCpu->hm.s.Event.fPending);
7016
7017 uint8_t uVector;
7018 TRPMEVENT enmTrpmEvent;
7019 RTGCUINT uErrCode;
7020 RTGCUINTPTR GCPtrFaultAddress;
7021 uint8_t cbInstr;
7022
7023 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7024 AssertRC(rc);
7025
7026 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7027 uint32_t u32IntInfo = uVector | VMX_EXIT_INT_INFO_VALID;
7028 if (enmTrpmEvent == TRPM_TRAP)
7029 {
7030 switch (uVector)
7031 {
7032 case X86_XCPT_NMI:
7033 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7034 break;
7035
7036 case X86_XCPT_BP:
7037 case X86_XCPT_OF:
7038 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7039 break;
7040
7041 case X86_XCPT_PF:
7042 case X86_XCPT_DF:
7043 case X86_XCPT_TS:
7044 case X86_XCPT_NP:
7045 case X86_XCPT_SS:
7046 case X86_XCPT_GP:
7047 case X86_XCPT_AC:
7048 u32IntInfo |= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7049 RT_FALL_THRU();
7050 default:
7051 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7052 break;
7053 }
7054 }
7055 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7056 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7057 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7058 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7059 else
7060 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7061
7062 rc = TRPMResetTrap(pVCpu);
7063 AssertRC(rc);
7064 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7065 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7066
7067 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7068}
7069
7070
7071/**
7072 * Converts the pending HM event into a TRPM trap.
7073 *
7074 * @param pVCpu The cross context virtual CPU structure.
7075 */
7076static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7077{
7078 Assert(pVCpu->hm.s.Event.fPending);
7079
7080 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7081 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7082 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
7083 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7084
7085 /* If a trap was already pending, we did something wrong! */
7086 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7087
7088 TRPMEVENT enmTrapType;
7089 switch (uVectorType)
7090 {
7091 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7092 enmTrapType = TRPM_HARDWARE_INT;
7093 break;
7094
7095 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7096 enmTrapType = TRPM_SOFTWARE_INT;
7097 break;
7098
7099 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7100 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7101 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7102 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7103 enmTrapType = TRPM_TRAP;
7104 break;
7105
7106 default:
7107 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7108 enmTrapType = TRPM_32BIT_HACK;
7109 break;
7110 }
7111
7112 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7113
7114 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7115 AssertRC(rc);
7116
7117 if (fErrorCodeValid)
7118 TRPMSetErrorCode(pVCpu, uErrorCode);
7119
7120 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7121 && uVector == X86_XCPT_PF)
7122 {
7123 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7124 }
7125 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7126 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7127 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7128 {
7129 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7130 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
7131 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7132 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7133 }
7134
7135 /* Clear any pending events from the VMCS. */
7136 VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7137 VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, 0);
7138
7139 /* We're now done converting the pending event. */
7140 pVCpu->hm.s.Event.fPending = false;
7141}
7142
7143
7144/**
7145 * Does the necessary state syncing before returning to ring-3 for any reason
7146 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7147 *
7148 * @returns VBox status code.
7149 * @param pVCpu The cross context virtual CPU structure.
7150 * @param fImportState Whether to import the guest state from the VMCS back
7151 * to the guest-CPU context.
7152 *
7153 * @remarks No-long-jmp zone!!!
7154 */
7155static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
7156{
7157 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7158 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7159
7160 RTCPUID idCpu = RTMpCpuId();
7161 Log4Func(("HostCpuId=%u\n", idCpu));
7162
7163 /*
7164 * !!! IMPORTANT !!!
7165 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7166 */
7167
7168 /* Save the guest state if necessary. */
7169 if (fImportState)
7170 {
7171 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
7172 AssertRCReturn(rc, rc);
7173 }
7174
7175 /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
7176 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7177 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
7178
7179 /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
7180#ifdef VBOX_STRICT
7181 if (CPUMIsHyperDebugStateActive(pVCpu))
7182 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
7183#endif
7184 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7185 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7186 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7187
7188#if HC_ARCH_BITS == 64
7189 /* Restore host-state bits that VT-x only restores partially. */
7190 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7191 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7192 {
7193 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7194 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7195 }
7196 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7197#endif
7198
7199 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7200 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7201 {
7202 /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
7203 if (!fImportState)
7204 {
7205 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS);
7206 AssertRCReturn(rc, rc);
7207 }
7208 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7209 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7210 }
7211 else
7212 pVCpu->hm.s.vmx.fLazyMsrs = 0;
7213
7214 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7215 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7216
7217 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7218 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
7219 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
7220 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
7221 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
7222 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7223 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7224 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7225 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7226
7227 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7228
7229 /** @todo This partially defeats the purpose of having preemption hooks.
7230 * The problem is, deregistering the hooks should be moved to a place that
7231 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7232 * context.
7233 */
7234 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7235 {
7236 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7237 AssertRCReturn(rc, rc);
7238
7239 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7240 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7241 }
7242 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7243 NOREF(idCpu);
7244
7245 return VINF_SUCCESS;
7246}
7247
7248
7249/**
7250 * Leaves the VT-x session.
7251 *
7252 * @returns VBox status code.
7253 * @param pVCpu The cross context virtual CPU structure.
7254 *
7255 * @remarks No-long-jmp zone!!!
7256 */
7257static int hmR0VmxLeaveSession(PVMCPU pVCpu)
7258{
7259 HM_DISABLE_PREEMPT(pVCpu);
7260 HMVMX_ASSERT_CPU_SAFE(pVCpu);
7261 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7262 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7263
7264 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7265 and done this from the VMXR0ThreadCtxCallback(). */
7266 if (!pVCpu->hm.s.fLeaveDone)
7267 {
7268 int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
7269 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7270 pVCpu->hm.s.fLeaveDone = true;
7271 }
7272 Assert(!pVCpu->cpum.GstCtx.fExtrn);
7273
7274 /*
7275 * !!! IMPORTANT !!!
7276 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7277 */
7278
7279 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7280 /** @todo Deregistering here means we need to VMCLEAR always
7281 * (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
7282 * for calling VMMR0ThreadCtxHookDisable here! */
7283 VMMR0ThreadCtxHookDisable(pVCpu);
7284
7285 /* Leave HM context. This takes care of local init (term). */
7286 int rc = HMR0LeaveCpu(pVCpu);
7287
7288 HM_RESTORE_PREEMPT();
7289 return rc;
7290}
7291
7292
7293/**
7294 * Does the necessary state syncing before doing a longjmp to ring-3.
7295 *
7296 * @returns VBox status code.
7297 * @param pVCpu The cross context virtual CPU structure.
7298 *
7299 * @remarks No-long-jmp zone!!!
7300 */
7301DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
7302{
7303 return hmR0VmxLeaveSession(pVCpu);
7304}
7305
7306
7307/**
7308 * Take necessary actions before going back to ring-3.
7309 *
7310 * An action requires us to go back to ring-3. This function does the necessary
7311 * steps before we can safely return to ring-3. This is not the same as longjmps
7312 * to ring-3, this is voluntary and prepares the guest so it may continue
7313 * executing outside HM (recompiler/IEM).
7314 *
7315 * @returns VBox status code.
7316 * @param pVCpu The cross context virtual CPU structure.
7317 * @param rcExit The reason for exiting to ring-3. Can be
7318 * VINF_VMM_UNKNOWN_RING3_CALL.
7319 */
7320static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
7321{
7322 Assert(pVCpu);
7323 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7324
7325 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7326 {
7327 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7328 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7329 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7330 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7331 }
7332
7333 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7334 VMMRZCallRing3Disable(pVCpu);
7335 Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
7336
7337 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7338 if (pVCpu->hm.s.Event.fPending)
7339 {
7340 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7341 Assert(!pVCpu->hm.s.Event.fPending);
7342 }
7343
7344 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7345 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7346
7347 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7348 and if we're injecting an event we should have a TRPM trap pending. */
7349 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7350#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
7351 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7352#endif
7353
7354 /* Save guest state and restore host state bits. */
7355 int rc = hmR0VmxLeaveSession(pVCpu);
7356 AssertRCReturn(rc, rc);
7357 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7358 /* Thread-context hooks are unregistered at this point!!! */
7359
7360 /* Sync recompiler state. */
7361 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7362 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7363 | CPUM_CHANGED_LDTR
7364 | CPUM_CHANGED_GDTR
7365 | CPUM_CHANGED_IDTR
7366 | CPUM_CHANGED_TR
7367 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7368 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
7369 && CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
7370 {
7371 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7372 }
7373
7374 Assert(!pVCpu->hm.s.fClearTrapFlag);
7375
7376 /* Update the exit-to-ring 3 reason. */
7377 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
7378
7379 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7380 if (rcExit != VINF_EM_RAW_INTERRUPT)
7381 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7382
7383 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7384
7385 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7386 VMMRZCallRing3RemoveNotification(pVCpu);
7387 VMMRZCallRing3Enable(pVCpu);
7388
7389 return rc;
7390}
7391
7392
7393/**
7394 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7395 * longjump to ring-3 and possibly get preempted.
7396 *
7397 * @returns VBox status code.
7398 * @param pVCpu The cross context virtual CPU structure.
7399 * @param enmOperation The operation causing the ring-3 longjump.
7400 * @param pvUser User argument, currently unused, NULL.
7401 */
7402static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7403{
7404 RT_NOREF(pvUser);
7405 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7406 {
7407 /*
7408 * !!! IMPORTANT !!!
7409 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7410 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7411 */
7412 VMMRZCallRing3RemoveNotification(pVCpu);
7413 VMMRZCallRing3Disable(pVCpu);
7414 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7415 RTThreadPreemptDisable(&PreemptState);
7416
7417 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
7418 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7419 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7420
7421#if HC_ARCH_BITS == 64
7422 /* Restore host-state bits that VT-x only restores partially. */
7423 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7424 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7425 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7426 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7427#endif
7428
7429 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7430 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7431 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7432
7433 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7434 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7435 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7436 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7437 {
7438 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7439 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7440 }
7441
7442 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7443 VMMR0ThreadCtxHookDisable(pVCpu);
7444 HMR0LeaveCpu(pVCpu);
7445 RTThreadPreemptRestore(&PreemptState);
7446 return VINF_SUCCESS;
7447 }
7448
7449 Assert(pVCpu);
7450 Assert(pvUser);
7451 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7452 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7453
7454 VMMRZCallRing3Disable(pVCpu);
7455 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7456
7457 Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7458
7459 int rc = hmR0VmxLongJmpToRing3(pVCpu);
7460 AssertRCReturn(rc, rc);
7461
7462 VMMRZCallRing3Enable(pVCpu);
7463 return VINF_SUCCESS;
7464}
7465
7466
7467/**
7468 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7469 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7470 *
7471 * @param pVCpu The cross context virtual CPU structure.
7472 */
7473DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7474{
7475 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7476 {
7477 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7478 {
7479 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_PROC_CTLS_INT_WINDOW_EXIT;
7480 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7481 AssertRC(rc);
7482 Log4Func(("Setup interrupt-window exiting\n"));
7483 }
7484 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7485}
7486
7487
7488/**
7489 * Clears the interrupt-window exiting control in the VMCS.
7490 *
7491 * @param pVCpu The cross context virtual CPU structure.
7492 */
7493DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7494{
7495 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT);
7496 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
7497 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7498 AssertRC(rc);
7499 Log4Func(("Cleared interrupt-window exiting\n"));
7500}
7501
7502
7503/**
7504 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7505 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7506 *
7507 * @param pVCpu The cross context virtual CPU structure.
7508 */
7509DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7510{
7511 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7512 {
7513 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7514 {
7515 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7516 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7517 AssertRC(rc);
7518 Log4Func(("Setup NMI-window exiting\n"));
7519 }
7520 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7521}
7522
7523
7524/**
7525 * Clears the NMI-window exiting control in the VMCS.
7526 *
7527 * @param pVCpu The cross context virtual CPU structure.
7528 */
7529DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7530{
7531 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT);
7532 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7533 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7534 AssertRC(rc);
7535 Log4Func(("Cleared NMI-window exiting\n"));
7536}
7537
7538
7539/**
7540 * Evaluates the event to be delivered to the guest and sets it as the pending
7541 * event.
7542 *
7543 * @returns The VT-x guest-interruptibility state.
7544 * @param pVCpu The cross context virtual CPU structure.
7545 */
7546static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu)
7547{
7548 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7549 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7550 uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu);
7551 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7552 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7553 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7554
7555 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7556 Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7557 Assert(!fBlockSti || pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7558 Assert(!TRPMHasTrap(pVCpu));
7559
7560 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7561 APICUpdatePendingInterrupts(pVCpu);
7562
7563 /*
7564 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7565 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7566 */
7567 /** @todo SMI. SMIs take priority over NMIs. */
7568 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7569 {
7570 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7571 if ( !pVCpu->hm.s.Event.fPending
7572 && !fBlockNmi
7573 && !fBlockSti
7574 && !fBlockMovSS)
7575 {
7576 Log4Func(("Pending NMI\n"));
7577 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INT_INFO_VALID;
7578 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7579
7580 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7581 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7582 }
7583 else
7584 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7585 }
7586 /*
7587 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7588 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7589 */
7590 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7591 && !pVCpu->hm.s.fSingleInstruction)
7592 {
7593 Assert(!DBGFIsStepping(pVCpu));
7594 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7595 AssertRCReturn(rc, 0);
7596 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7597 if ( !pVCpu->hm.s.Event.fPending
7598 && !fBlockInt
7599 && !fBlockSti
7600 && !fBlockMovSS)
7601 {
7602 uint8_t u8Interrupt;
7603 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7604 if (RT_SUCCESS(rc))
7605 {
7606 Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7607 uint32_t u32IntInfo = u8Interrupt
7608 | VMX_EXIT_INT_INFO_VALID
7609 | (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7610
7611 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7612 }
7613 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7614 {
7615 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
7616 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7617 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7618
7619 /*
7620 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7621 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7622 * need to re-set this force-flag here.
7623 */
7624 }
7625 else
7626 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7627 }
7628 else
7629 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7630 }
7631
7632 return fIntrState;
7633}
7634
7635
7636/**
7637 * Sets a pending-debug exception to be delivered to the guest if the guest is
7638 * single-stepping in the VMCS.
7639 *
7640 * @param pVCpu The cross context virtual CPU structure.
7641 */
7642DECLINLINE(int) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7643{
7644 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7645 RT_NOREF(pVCpu);
7646 return VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
7647}
7648
7649
7650/**
7651 * Injects any pending events into the guest if the guest is in a state to
7652 * receive them.
7653 *
7654 * @returns Strict VBox status code (i.e. informational status codes too).
7655 * @param pVCpu The cross context virtual CPU structure.
7656 * @param fIntrState The VT-x guest-interruptibility state.
7657 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7658 * return VINF_EM_DBG_STEPPED if the event was
7659 * dispatched directly.
7660 */
7661static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, uint32_t fIntrState, bool fStepping)
7662{
7663 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7664 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7665
7666 bool fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7667 bool fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7668
7669 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7670 Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7671 Assert(!fBlockSti || pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7672 Assert(!TRPMHasTrap(pVCpu));
7673
7674 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7675 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7676 if (pVCpu->hm.s.Event.fPending)
7677 {
7678 /*
7679 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7680 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7681 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7682 *
7683 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7684 */
7685 uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7686#ifdef VBOX_STRICT
7687 if (uIntType == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
7688 {
7689 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7690 Assert(!fBlockInt);
7691 Assert(!fBlockSti);
7692 Assert(!fBlockMovSS);
7693 }
7694 else if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
7695 {
7696 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7697 Assert(!fBlockSti);
7698 Assert(!fBlockMovSS);
7699 Assert(!fBlockNmi);
7700 }
7701#endif
7702 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7703 uIntType));
7704 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7705 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7706 &fIntrState);
7707 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7708
7709 /* Update the interruptibility-state as it could have been changed by
7710 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7711 fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7712 fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7713
7714 if (uIntType == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
7715 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7716 else
7717 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7718 }
7719
7720 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7721 if ( fBlockSti
7722 || fBlockMovSS)
7723 {
7724 if (!pVCpu->hm.s.fSingleInstruction)
7725 {
7726 /*
7727 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7728 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7729 * See Intel spec. 27.3.4 "Saving Non-Register State".
7730 */
7731 Assert(!DBGFIsStepping(pVCpu));
7732 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7733 AssertRCReturn(rc, rc);
7734 if (pCtx->eflags.Bits.u1TF)
7735 {
7736 int rc2 = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7737 AssertRCReturn(rc2, rc2);
7738 }
7739 }
7740 else if (pCtx->eflags.Bits.u1TF)
7741 {
7742 /*
7743 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7744 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7745 */
7746 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
7747 fIntrState = 0;
7748 }
7749 }
7750
7751 /*
7752 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7753 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7754 */
7755 int rc3 = hmR0VmxExportGuestIntrState(pVCpu, fIntrState);
7756 AssertRCReturn(rc3, rc3);
7757
7758 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7759 NOREF(fBlockMovSS); NOREF(fBlockSti);
7760 return rcStrict;
7761}
7762
7763
7764/**
7765 * Injects a double-fault (\#DF) exception into the VM.
7766 *
7767 * @returns Strict VBox status code (i.e. informational status codes too).
7768 * @param pVCpu The cross context virtual CPU structure.
7769 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7770 * and should return VINF_EM_DBG_STEPPED if the event
7771 * is injected directly (register modified by us, not
7772 * by hardware on VM-entry).
7773 * @param pfIntrState Pointer to the current guest interruptibility-state.
7774 * This interruptibility-state will be updated if
7775 * necessary. This cannot not be NULL.
7776 */
7777DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, bool fStepping, uint32_t *pfIntrState)
7778{
7779 uint32_t const u32IntInfo = X86_XCPT_DF | VMX_EXIT_INT_INFO_VALID
7780 | (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT)
7781 | VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7782 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */, fStepping,
7783 pfIntrState);
7784}
7785
7786
7787/**
7788 * Injects a general-protection (\#GP) fault into the VM.
7789 *
7790 * @returns Strict VBox status code (i.e. informational status codes too).
7791 * @param pVCpu The cross context virtual CPU structure.
7792 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7793 * mode, i.e. in real-mode it's not valid).
7794 * @param u32ErrorCode The error code associated with the \#GP.
7795 * @param fStepping Whether we're running in
7796 * hmR0VmxRunGuestCodeStep() and should return
7797 * VINF_EM_DBG_STEPPED if the event is injected
7798 * directly (register modified by us, not by
7799 * hardware on VM-entry).
7800 * @param pfIntrState Pointer to the current guest interruptibility-state.
7801 * This interruptibility-state will be updated if
7802 * necessary. This cannot not be NULL.
7803 */
7804DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, bool fErrorCodeValid, uint32_t u32ErrorCode, bool fStepping,
7805 uint32_t *pfIntrState)
7806{
7807 uint32_t const u32IntInfo = X86_XCPT_GP | VMX_EXIT_INT_INFO_VALID
7808 | (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT)
7809 | (fErrorCodeValid ? VMX_EXIT_INT_INFO_ERROR_CODE_VALID : 0);
7810 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */, fStepping,
7811 pfIntrState);
7812}
7813
7814
7815/**
7816 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7817 * stack.
7818 *
7819 * @returns Strict VBox status code (i.e. informational status codes too).
7820 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7821 * @param pVCpu The cross context virtual CPU structure.
7822 * @param uValue The value to push to the guest stack.
7823 */
7824static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
7825{
7826 /*
7827 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7828 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7829 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7830 */
7831 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7832 if (pCtx->sp == 1)
7833 return VINF_EM_RESET;
7834 pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7835 int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
7836 AssertRC(rc);
7837 return rc;
7838}
7839
7840
7841/**
7842 * Injects an event into the guest upon VM-entry by updating the relevant fields
7843 * in the VM-entry area in the VMCS.
7844 *
7845 * @returns Strict VBox status code (i.e. informational status codes too).
7846 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7847 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7848 *
7849 * @param pVCpu The cross context virtual CPU structure.
7850 * @param u64IntInfo The VM-entry interruption-information field.
7851 * @param cbInstr The VM-entry instruction length in bytes (for
7852 * software interrupts, exceptions and privileged
7853 * software exceptions).
7854 * @param u32ErrCode The VM-entry exception error code.
7855 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7856 * @param pfIntrState Pointer to the current guest interruptibility-state.
7857 * This interruptibility-state will be updated if
7858 * necessary. This cannot not be NULL.
7859 * @param fStepping Whether we're running in
7860 * hmR0VmxRunGuestCodeStep() and should return
7861 * VINF_EM_DBG_STEPPED if the event is injected
7862 * directly (register modified by us, not by
7863 * hardware on VM-entry).
7864 */
7865static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7866 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7867{
7868 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7869 AssertMsg(!RT_HI_U32(u64IntInfo), ("%#RX64\n", u64IntInfo));
7870 Assert(pfIntrState);
7871
7872 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7873 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7874 uint32_t const uVector = VMX_EXIT_INT_INFO_VECTOR(u32IntInfo);
7875 uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(u32IntInfo);
7876
7877#ifdef VBOX_STRICT
7878 /*
7879 * Validate the error-code-valid bit for hardware exceptions.
7880 * No error codes for exceptions in real-mode.
7881 *
7882 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7883 */
7884 if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
7885 && !CPUMIsGuestInRealModeEx(pCtx))
7886 {
7887 switch (uVector)
7888 {
7889 case X86_XCPT_PF:
7890 case X86_XCPT_DF:
7891 case X86_XCPT_TS:
7892 case X86_XCPT_NP:
7893 case X86_XCPT_SS:
7894 case X86_XCPT_GP:
7895 case X86_XCPT_AC:
7896 AssertMsg(VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
7897 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7898 RT_FALL_THRU();
7899 default:
7900 break;
7901 }
7902 }
7903#endif
7904
7905 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7906 Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
7907 || !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
7908
7909 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7910
7911 /*
7912 * Hardware interrupts & exceptions cannot be delivered through the software interrupt
7913 * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7914 * interrupt handler in the (real-mode) guest.
7915 *
7916 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7917 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7918 */
7919 if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
7920 {
7921 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7922 {
7923 /*
7924 * For unrestricted execution enabled CPUs running real-mode guests, we must not
7925 * set the deliver-error-code bit.
7926 *
7927 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7928 */
7929 u32IntInfo &= ~VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7930 }
7931 else
7932 {
7933 PVM pVM = pVCpu->CTX_SUFF(pVM);
7934 Assert(PDMVmmDevHeapIsEnabled(pVM));
7935 Assert(pVM->hm.s.vmx.pRealModeTSS);
7936
7937 /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7938 int rc2 = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_TABLE_MASK | CPUMCTX_EXTRN_RIP
7939 | CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_RFLAGS);
7940 AssertRCReturn(rc2, rc2);
7941
7942 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7943 size_t const cbIdtEntry = sizeof(X86IDTR16);
7944 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
7945 {
7946 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7947 if (uVector == X86_XCPT_DF)
7948 return VINF_EM_RESET;
7949
7950 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7951 if (uVector == X86_XCPT_GP)
7952 return hmR0VmxInjectXcptDF(pVCpu, fStepping, pfIntrState);
7953
7954 /*
7955 * If we're injecting an event with no valid IDT entry, inject a #GP.
7956 * No error codes for exceptions in real-mode.
7957 *
7958 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7959 */
7960 return hmR0VmxInjectXcptGP(pVCpu, false /* fErrCodeValid */, 0 /* u32ErrCode */, fStepping, pfIntrState);
7961 }
7962
7963 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7964 uint16_t uGuestIp = pCtx->ip;
7965 if (uIntType == VMX_EXIT_INT_INFO_TYPE_SW_XCPT)
7966 {
7967 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7968 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7969 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7970 }
7971 else if (uIntType == VMX_EXIT_INT_INFO_TYPE_SW_INT)
7972 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7973
7974 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7975 X86IDTR16 IdtEntry;
7976 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
7977 rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7978 AssertRCReturn(rc2, rc2);
7979
7980 /* Construct the stack frame for the interrupt/exception handler. */
7981 VBOXSTRICTRC rcStrict;
7982 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
7983 if (rcStrict == VINF_SUCCESS)
7984 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
7985 if (rcStrict == VINF_SUCCESS)
7986 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
7987
7988 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7989 if (rcStrict == VINF_SUCCESS)
7990 {
7991 pCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7992 pCtx->rip = IdtEntry.offSel;
7993 pCtx->cs.Sel = IdtEntry.uSel;
7994 pCtx->cs.ValidSel = IdtEntry.uSel;
7995 pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7996 if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
7997 && uVector == X86_XCPT_PF)
7998 pCtx->cr2 = GCPtrFaultAddress;
7999
8000 /* If any other guest-state bits are changed here, make sure to update
8001 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
8002 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS | HM_CHANGED_GUEST_CR2
8003 | HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
8004 | HM_CHANGED_GUEST_RSP);
8005
8006 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8007 if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8008 {
8009 Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8010 && uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
8011 Log4Func(("Clearing inhibition due to STI\n"));
8012 *pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8013 }
8014 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8015 u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8016
8017 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8018 it, if we are returning to ring-3 before executing guest code. */
8019 pVCpu->hm.s.Event.fPending = false;
8020
8021 /* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
8022 if (fStepping)
8023 rcStrict = VINF_EM_DBG_STEPPED;
8024 }
8025 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8026 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8027 return rcStrict;
8028 }
8029 }
8030
8031 /* Validate. */
8032 Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8033 Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8034
8035 /* Inject. */
8036 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8037 if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8038 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8039 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8040 AssertRCReturn(rc, rc);
8041
8042 /* Update CR2. */
8043 if ( VMX_EXIT_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8044 && uVector == X86_XCPT_PF)
8045 pCtx->cr2 = GCPtrFaultAddress;
8046
8047 Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8048
8049 return VINF_SUCCESS;
8050}
8051
8052
8053/**
8054 * Clears the interrupt-window exiting control in the VMCS and if necessary
8055 * clears the current event in the VMCS as well.
8056 *
8057 * @returns VBox status code.
8058 * @param pVCpu The cross context virtual CPU structure.
8059 *
8060 * @remarks Use this function only to clear events that have not yet been
8061 * delivered to the guest but are injected in the VMCS!
8062 * @remarks No-long-jump zone!!!
8063 */
8064static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8065{
8066 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8067 {
8068 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8069 Log4Func(("Cleared interrupt widow\n"));
8070 }
8071
8072 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8073 {
8074 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8075 Log4Func(("Cleared interrupt widow\n"));
8076 }
8077}
8078
8079
8080/**
8081 * Enters the VT-x session.
8082 *
8083 * @returns VBox status code.
8084 * @param pVCpu The cross context virtual CPU structure.
8085 * @param pHostCpu Pointer to the global CPU info struct.
8086 */
8087VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu, PHMGLOBALCPUINFO pHostCpu)
8088{
8089 AssertPtr(pVCpu);
8090 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8091 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8092 RT_NOREF(pHostCpu);
8093
8094 LogFlowFunc(("pVCpu=%p\n", pVCpu));
8095 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8096 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8097
8098#ifdef VBOX_STRICT
8099 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8100 RTCCUINTREG uHostCR4 = ASMGetCR4();
8101 if (!(uHostCR4 & X86_CR4_VMXE))
8102 {
8103 LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8104 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8105 }
8106#endif
8107
8108 /*
8109 * Load the VCPU's VMCS as the current (and active) one.
8110 */
8111 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8112 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8113 if (RT_FAILURE(rc))
8114 return rc;
8115
8116 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8117 pVCpu->hm.s.fLeaveDone = false;
8118 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8119
8120 return VINF_SUCCESS;
8121}
8122
8123
8124/**
8125 * The thread-context callback (only on platforms which support it).
8126 *
8127 * @param enmEvent The thread-context event.
8128 * @param pVCpu The cross context virtual CPU structure.
8129 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8130 * @thread EMT(pVCpu)
8131 */
8132VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8133{
8134 NOREF(fGlobalInit);
8135
8136 switch (enmEvent)
8137 {
8138 case RTTHREADCTXEVENT_OUT:
8139 {
8140 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8141 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8142 VMCPU_ASSERT_EMT(pVCpu);
8143
8144 /* No longjmps (logger flushes, locks) in this fragile context. */
8145 VMMRZCallRing3Disable(pVCpu);
8146 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8147
8148 /*
8149 * Restore host-state (FPU, debug etc.)
8150 */
8151 if (!pVCpu->hm.s.fLeaveDone)
8152 {
8153 /*
8154 * Do -not- import the guest-state here as we might already be in the middle of importing
8155 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
8156 */
8157 hmR0VmxLeave(pVCpu, false /* fImportState */);
8158 pVCpu->hm.s.fLeaveDone = true;
8159 }
8160
8161 /* Leave HM context, takes care of local init (term). */
8162 int rc = HMR0LeaveCpu(pVCpu);
8163 AssertRC(rc); NOREF(rc);
8164
8165 /* Restore longjmp state. */
8166 VMMRZCallRing3Enable(pVCpu);
8167 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8168 break;
8169 }
8170
8171 case RTTHREADCTXEVENT_IN:
8172 {
8173 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8174 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8175 VMCPU_ASSERT_EMT(pVCpu);
8176
8177 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8178 VMMRZCallRing3Disable(pVCpu);
8179 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8180
8181 /* Initialize the bare minimum state required for HM. This takes care of
8182 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8183 int rc = hmR0EnterCpu(pVCpu);
8184 AssertRC(rc);
8185 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8186 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8187
8188 /* Load the active VMCS as the current one. */
8189 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8190 {
8191 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8192 AssertRC(rc); NOREF(rc);
8193 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8194 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8195 }
8196 pVCpu->hm.s.fLeaveDone = false;
8197
8198 /* Restore longjmp state. */
8199 VMMRZCallRing3Enable(pVCpu);
8200 break;
8201 }
8202
8203 default:
8204 break;
8205 }
8206}
8207
8208
8209/**
8210 * Exports the host state into the VMCS host-state area.
8211 * Sets up the VM-exit MSR-load area.
8212 *
8213 * The CPU state will be loaded from these fields on every successful VM-exit.
8214 *
8215 * @returns VBox status code.
8216 * @param pVCpu The cross context virtual CPU structure.
8217 *
8218 * @remarks No-long-jump zone!!!
8219 */
8220static int hmR0VmxExportHostState(PVMCPU pVCpu)
8221{
8222 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8223
8224 int rc = VINF_SUCCESS;
8225 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8226 {
8227 rc = hmR0VmxExportHostControlRegs();
8228 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8229
8230 rc = hmR0VmxExportHostSegmentRegs(pVCpu);
8231 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8232
8233 rc = hmR0VmxExportHostMsrs(pVCpu);
8234 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8235
8236 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
8237 }
8238 return rc;
8239}
8240
8241
8242/**
8243 * Saves the host state in the VMCS host-state.
8244 *
8245 * @returns VBox status code.
8246 * @param pVCpu The cross context virtual CPU structure.
8247 *
8248 * @remarks No-long-jump zone!!!
8249 */
8250VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
8251{
8252 AssertPtr(pVCpu);
8253 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8254
8255 /*
8256 * Export the host state here while entering HM context.
8257 * When thread-context hooks are used, we might get preempted and have to re-save the host
8258 * state but most of the time we won't be, so do it here before we disable interrupts.
8259 */
8260 return hmR0VmxExportHostState(pVCpu);
8261}
8262
8263
8264/**
8265 * Exports the guest state into the VMCS guest-state area.
8266 *
8267 * The will typically be done before VM-entry when the guest-CPU state and the
8268 * VMCS state may potentially be out of sync.
8269 *
8270 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8271 * VM-entry controls.
8272 * Sets up the appropriate VMX non-root function to execute guest code based on
8273 * the guest CPU mode.
8274 *
8275 * @returns VBox strict status code.
8276 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8277 * without unrestricted guest access and the VMMDev is not presently
8278 * mapped (e.g. EFI32).
8279 *
8280 * @param pVCpu The cross context virtual CPU structure.
8281 *
8282 * @remarks No-long-jump zone!!!
8283 */
8284static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu)
8285{
8286 AssertPtr(pVCpu);
8287 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8288
8289 LogFlowFunc(("pVCpu=%p\n", pVCpu));
8290
8291 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
8292
8293 /* Determine real-on-v86 mode. */
8294 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8295 if ( !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
8296 && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
8297 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8298
8299 /*
8300 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8301 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8302 */
8303 int rc = hmR0VmxSelectVMRunHandler(pVCpu);
8304 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8305
8306 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8307 rc = hmR0VmxExportGuestEntryCtls(pVCpu);
8308 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8309
8310 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8311 rc = hmR0VmxExportGuestExitCtls(pVCpu);
8312 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8313
8314 rc = hmR0VmxExportGuestCR0(pVCpu);
8315 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8316
8317 VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu);
8318 if (rcStrict == VINF_SUCCESS)
8319 { /* likely */ }
8320 else
8321 {
8322 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8323 return rcStrict;
8324 }
8325
8326 rc = hmR0VmxExportGuestSegmentRegs(pVCpu);
8327 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8328
8329 /* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8330 may alter controls if we determine we don't have to swap EFER after all. */
8331 rc = hmR0VmxExportGuestMsrs(pVCpu);
8332 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8333
8334 rc = hmR0VmxExportGuestApicTpr(pVCpu);
8335 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8336
8337 rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8338 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8339
8340 /* Exporting RFLAGS here is fine, even though RFLAGS.TF might depend on guest debug state which is
8341 not exported here. It is re-evaluated and updated if necessary in hmR0VmxExportSharedState(). */
8342 rc = hmR0VmxExportGuestRip(pVCpu);
8343 rc |= hmR0VmxExportGuestRsp(pVCpu);
8344 rc |= hmR0VmxExportGuestRflags(pVCpu);
8345 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8346
8347 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8348 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8349 | HM_CHANGED_GUEST_CR2
8350 | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8351 | HM_CHANGED_GUEST_X87
8352 | HM_CHANGED_GUEST_SSE_AVX
8353 | HM_CHANGED_GUEST_OTHER_XSAVE
8354 | HM_CHANGED_GUEST_XCRx
8355 | HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8356 | HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8357 | HM_CHANGED_GUEST_TSC_AUX
8358 | HM_CHANGED_GUEST_OTHER_MSRS
8359 | HM_CHANGED_GUEST_HWVIRT
8360 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8361
8362 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8363 return rc;
8364}
8365
8366
8367/**
8368 * Exports the state shared between the host and guest into the VMCS.
8369 *
8370 * @param pVCpu The cross context virtual CPU structure.
8371 *
8372 * @remarks No-long-jump zone!!!
8373 */
8374static void hmR0VmxExportSharedState(PVMCPU pVCpu)
8375{
8376 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8377 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8378
8379 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8380 {
8381 int rc = hmR0VmxExportSharedDebugState(pVCpu);
8382 AssertRC(rc);
8383 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8384
8385 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8386 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8387 {
8388 rc = hmR0VmxExportGuestRflags(pVCpu);
8389 AssertRC(rc);
8390 }
8391 }
8392
8393 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8394 {
8395 hmR0VmxLazyLoadGuestMsrs(pVCpu);
8396 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8397 }
8398
8399 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8400 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8401}
8402
8403
8404/**
8405 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8406 *
8407 * @returns Strict VBox status code (i.e. informational status codes too).
8408 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8409 * without unrestricted guest access and the VMMDev is not presently
8410 * mapped (e.g. EFI32).
8411 *
8412 * @param pVCpu The cross context virtual CPU structure.
8413 *
8414 * @remarks No-long-jump zone!!!
8415 */
8416static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu)
8417{
8418 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8419 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8420 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8421
8422#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8423 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8424#endif
8425
8426 /*
8427 * For many exits it's only RIP that changes and hence try to export it first
8428 * without going through a lot of change flag checks.
8429 */
8430 VBOXSTRICTRC rcStrict;
8431 uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8432 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8433 if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8434 {
8435 rcStrict = hmR0VmxExportGuestRip(pVCpu);
8436 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8437 { /* likely */}
8438 else
8439 AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8440 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8441 }
8442 else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8443 {
8444 rcStrict = hmR0VmxExportGuestState(pVCpu);
8445 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8446 { /* likely */}
8447 else
8448 {
8449 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8450 VBOXSTRICTRC_VAL(rcStrict)));
8451 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8452 return rcStrict;
8453 }
8454 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8455 }
8456 else
8457 rcStrict = VINF_SUCCESS;
8458
8459#ifdef VBOX_STRICT
8460 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8461 fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8462 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8463 AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8464 ("fCtxChanged=%#RX64\n", fCtxChanged));
8465#endif
8466 return rcStrict;
8467}
8468
8469
8470/**
8471 * Does the preparations before executing guest code in VT-x.
8472 *
8473 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8474 * recompiler/IEM. We must be cautious what we do here regarding committing
8475 * guest-state information into the VMCS assuming we assuredly execute the
8476 * guest in VT-x mode.
8477 *
8478 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8479 * the common-state (TRPM/forceflags), we must undo those changes so that the
8480 * recompiler/IEM can (and should) use them when it resumes guest execution.
8481 * Otherwise such operations must be done when we can no longer exit to ring-3.
8482 *
8483 * @returns Strict VBox status code (i.e. informational status codes too).
8484 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8485 * have been disabled.
8486 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8487 * double-fault into the guest.
8488 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8489 * dispatched directly.
8490 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8491 *
8492 * @param pVCpu The cross context virtual CPU structure.
8493 * @param pVmxTransient Pointer to the VMX transient structure.
8494 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8495 * us ignore some of the reasons for returning to
8496 * ring-3, and return VINF_EM_DBG_STEPPED if event
8497 * dispatching took place.
8498 */
8499static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
8500{
8501 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8502
8503#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_ONLY_IN_IEM
8504 Log2(("hmR0SvmPreRunGuest: Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
8505 return VINF_EM_RESCHEDULE_REM;
8506#endif
8507
8508#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8509 PGMRZDynMapFlushAutoSet(pVCpu);
8510#endif
8511
8512 /* Check force flag actions that might require us to go back to ring-3. */
8513 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
8514 if (rcStrict == VINF_SUCCESS)
8515 { /* FFs doesn't get set all the time. */ }
8516 else
8517 return rcStrict;
8518
8519 /*
8520 * Setup the virtualized-APIC accesses.
8521 *
8522 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8523 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8524 *
8525 * This is the reason we do it here and not in hmR0VmxExportGuestState().
8526 */
8527 PVM pVM = pVCpu->CTX_SUFF(pVM);
8528 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8529 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
8530 && PDMHasApic(pVM))
8531 {
8532 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8533 Assert(u64MsrApicBase);
8534 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8535
8536 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8537
8538 /* Unalias any existing mapping. */
8539 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8540 AssertRCReturn(rc, rc);
8541
8542 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8543 Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8544 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8545 AssertRCReturn(rc, rc);
8546
8547 /* Update the per-VCPU cache of the APIC base MSR. */
8548 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8549 }
8550
8551 if (TRPMHasTrap(pVCpu))
8552 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8553 uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu);
8554
8555 /*
8556 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
8557 * needs to be done with longjmps or interrupts + preemption enabled. Event injection might
8558 * also result in triple-faulting the VM.
8559 */
8560 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, fIntrState, fStepping);
8561 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8562 { /* likely */ }
8563 else
8564 {
8565 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8566 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8567 return rcStrict;
8568 }
8569
8570 /*
8571 * A longjump might result in importing CR3 even for VM-exits that don't necessarily
8572 * import CR3 themselves. We will need to update them here, as even as late as the above
8573 * hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
8574 * the below force flags to be set.
8575 */
8576 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8577 {
8578 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
8579 int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8580 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
8581 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
8582 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8583 }
8584 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8585 {
8586 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8587 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8588 }
8589
8590 /*
8591 * No longjmps to ring-3 from this point on!!!
8592 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8593 * This also disables flushing of the R0-logger instance (if any).
8594 */
8595 VMMRZCallRing3Disable(pVCpu);
8596
8597 /*
8598 * Export the guest state bits.
8599 *
8600 * We cannot perform longjmps while loading the guest state because we do not preserve the
8601 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8602 * CPU migration.
8603 *
8604 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8605 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8606 * Hence, loading of the guest state needs to be done -after- injection of events.
8607 */
8608 rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu);
8609 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8610 { /* likely */ }
8611 else
8612 {
8613 VMMRZCallRing3Enable(pVCpu);
8614 return rcStrict;
8615 }
8616
8617 /*
8618 * We disable interrupts so that we don't miss any interrupts that would flag preemption
8619 * (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
8620 * preemption disabled for a while. Since this is purly to aid the
8621 * RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
8622 * disable interrupt on NT.
8623 *
8624 * We need to check for force-flags that could've possible been altered since we last
8625 * checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
8626 * see @bugref{6398}).
8627 *
8628 * We also check a couple of other force-flags as a last opportunity to get the EMT back
8629 * to ring-3 before executing guest code.
8630 */
8631 pVmxTransient->fEFlags = ASMIntDisableFlags();
8632
8633 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8634 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8635 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8636 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8637 {
8638 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8639 {
8640 pVCpu->hm.s.Event.fPending = false;
8641
8642 /*
8643 * We've injected any pending events. This is really the point of no return (to ring-3).
8644 *
8645 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8646 * returns from this function, so don't enable them here.
8647 */
8648 return VINF_SUCCESS;
8649 }
8650
8651 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
8652 rcStrict = VINF_EM_RAW_INTERRUPT;
8653 }
8654 else
8655 {
8656 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8657 rcStrict = VINF_EM_RAW_TO_R3;
8658 }
8659
8660 ASMSetFlags(pVmxTransient->fEFlags);
8661 VMMRZCallRing3Enable(pVCpu);
8662
8663 return rcStrict;
8664}
8665
8666
8667/**
8668 * Prepares to run guest code in VT-x and we've committed to doing so. This
8669 * means there is no backing out to ring-3 or anywhere else at this
8670 * point.
8671 *
8672 * @param pVCpu The cross context virtual CPU structure.
8673 * @param pVmxTransient Pointer to the VMX transient structure.
8674 *
8675 * @remarks Called with preemption disabled.
8676 * @remarks No-long-jump zone!!!
8677 */
8678static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
8679{
8680 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8681 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8682 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8683
8684 /*
8685 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8686 */
8687 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8688 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8689
8690 PVM pVM = pVCpu->CTX_SUFF(pVM);
8691 if (!CPUMIsGuestFPUStateActive(pVCpu))
8692 {
8693 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8694 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8695 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
8696 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8697 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8698 }
8699
8700 /*
8701 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8702 */
8703 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8704 && pVCpu->hm.s.vmx.cMsrs > 0)
8705 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8706
8707 /*
8708 * Re-save the host state bits as we may've been preempted (only happens when
8709 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8710 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8711 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8712 * See @bugref{8432}.
8713 */
8714 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8715 {
8716 int rc = hmR0VmxExportHostState(pVCpu);
8717 AssertRC(rc);
8718 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8719 }
8720 Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8721
8722 /*
8723 * Export the state shared between host and guest (FPU, debug, lazy MSRs).
8724 */
8725 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8726 hmR0VmxExportSharedState(pVCpu);
8727 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8728
8729 /* Store status of the shared guest-host state at the time of VM-entry. */
8730#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8731 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
8732 {
8733 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8734 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8735 }
8736 else
8737#endif
8738 {
8739 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8740 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8741 }
8742
8743 /*
8744 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8745 */
8746 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
8747 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8748
8749 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8750 RTCPUID idCurrentCpu = pCpu->idCpu;
8751 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8752 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8753 {
8754 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8755 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8756 }
8757
8758 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8759 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8760 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8761 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8762
8763 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8764
8765 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8766 to start executing. */
8767
8768 /*
8769 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8770 */
8771 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
8772 {
8773 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
8774 {
8775 bool fMsrUpdated;
8776 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8777 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8778 &fMsrUpdated);
8779 AssertRC(rc2);
8780 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8781 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8782 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8783 }
8784 else
8785 {
8786 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8787 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8788 }
8789 }
8790
8791 if (pVM->cpum.ro.GuestFeatures.fIbrs)
8792 {
8793 bool fMsrUpdated;
8794 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8795 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8796 &fMsrUpdated);
8797 AssertRC(rc2);
8798 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8799 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8800 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8801 }
8802
8803#ifdef VBOX_STRICT
8804 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8805 hmR0VmxCheckHostEferMsr(pVCpu);
8806 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8807#endif
8808#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8809 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
8810 {
8811 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
8812 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8813 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8814 }
8815#endif
8816}
8817
8818
8819/**
8820 * Performs some essential restoration of state after running guest code in
8821 * VT-x.
8822 *
8823 * @param pVCpu The cross context virtual CPU structure.
8824 * @param pVmxTransient Pointer to the VMX transient structure.
8825 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8826 *
8827 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8828 *
8829 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8830 * unconditionally when it is safe to do so.
8831 */
8832static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8833{
8834 uint64_t const uHostTsc = ASMReadTSC();
8835 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8836
8837 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8838 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8839 pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8840 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8841 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8842 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8843
8844 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
8845 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TscOffset);
8846
8847 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
8848 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8849 Assert(!ASMIntAreEnabled());
8850 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8851
8852#if HC_ARCH_BITS == 64
8853 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8854#endif
8855#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8856 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8857 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8858 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8859#else
8860 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8861#endif
8862#ifdef VBOX_STRICT
8863 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8864#endif
8865 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8866
8867 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8868 uint32_t uExitReason;
8869 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8870 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8871 AssertRC(rc);
8872 pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
8873 pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
8874
8875 if (rcVMRun == VINF_SUCCESS)
8876 {
8877 /*
8878 * Update the VM-exit history array here even if the VM-entry failed due to:
8879 * - Invalid guest state.
8880 * - MSR loading.
8881 * - Machine-check event.
8882 *
8883 * In any of the above cases we will still have a "valid" VM-exit reason
8884 * despite @a fVMEntryFailed being false.
8885 *
8886 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8887 *
8888 * Note! We don't have CS or RIP at this point. Will probably address that later
8889 * by amending the history entry added here.
8890 */
8891 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8892 UINT64_MAX, uHostTsc);
8893
8894 if (!pVmxTransient->fVMEntryFailed)
8895 {
8896 VMMRZCallRing3Enable(pVCpu);
8897
8898 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8899 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8900
8901#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8902 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8903 AssertRC(rc);
8904#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8905 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8906 AssertRC(rc);
8907#else
8908 /*
8909 * Import the guest-interruptibility state always as we need it while evaluating
8910 * injecting events on re-entry.
8911 *
8912 * We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8913 * checking for real-mode while exporting the state because all bits that cause
8914 * mode changes wrt CR0 are intercepted.
8915 */
8916 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8917 AssertRC(rc);
8918#endif
8919
8920 /*
8921 * Sync the TPR shadow with our APIC state.
8922 */
8923 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
8924 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8925 {
8926 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8927 AssertRC(rc);
8928 ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8929 }
8930
8931 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8932 return;
8933 }
8934 }
8935 else
8936 Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8937
8938 VMMRZCallRing3Enable(pVCpu);
8939}
8940
8941
8942/**
8943 * Runs the guest code using VT-x the normal way.
8944 *
8945 * @returns VBox status code.
8946 * @param pVCpu The cross context virtual CPU structure.
8947 *
8948 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8949 */
8950static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu)
8951{
8952 VMXTRANSIENT VmxTransient;
8953 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8954 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8955 uint32_t cLoops = 0;
8956
8957 for (;; cLoops++)
8958 {
8959 Assert(!HMR0SuspendPending());
8960 HMVMX_ASSERT_CPU_SAFE(pVCpu);
8961
8962 /* Preparatory work for running guest code, this may force us to return
8963 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8964 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8965 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
8966 if (rcStrict != VINF_SUCCESS)
8967 break;
8968
8969 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
8970 int rcRun = hmR0VmxRunGuest(pVCpu);
8971
8972 /* Restore any residual host-state and save any bits shared between host
8973 and guest into the guest-CPU state. Re-enables interrupts! */
8974 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8975
8976 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8977 if (RT_SUCCESS(rcRun))
8978 { /* very likely */ }
8979 else
8980 {
8981 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
8982 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
8983 return rcRun;
8984 }
8985
8986 /* Profile the VM-exit. */
8987 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8988 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8989 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8990 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
8991 HMVMX_START_EXIT_DISPATCH_PROF();
8992
8993 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
8994
8995 /* Handle the VM-exit. */
8996#ifdef HMVMX_USE_FUNCTION_TABLE
8997 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
8998#else
8999 rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient, VmxTransient.uExitReason);
9000#endif
9001 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
9002 if (rcStrict == VINF_SUCCESS)
9003 {
9004 if (cLoops <= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
9005 continue; /* likely */
9006 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9007 rcStrict = VINF_EM_RAW_INTERRUPT;
9008 }
9009 break;
9010 }
9011
9012 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9013 return rcStrict;
9014}
9015
9016
9017
9018/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
9019 * probes.
9020 *
9021 * The following few functions and associated structure contains the bloat
9022 * necessary for providing detailed debug events and dtrace probes as well as
9023 * reliable host side single stepping. This works on the principle of
9024 * "subclassing" the normal execution loop and workers. We replace the loop
9025 * method completely and override selected helpers to add necessary adjustments
9026 * to their core operation.
9027 *
9028 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9029 * any performance for debug and analysis features.
9030 *
9031 * @{
9032 */
9033
9034/**
9035 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
9036 * the debug run loop.
9037 */
9038typedef struct VMXRUNDBGSTATE
9039{
9040 /** The RIP we started executing at. This is for detecting that we stepped. */
9041 uint64_t uRipStart;
9042 /** The CS we started executing with. */
9043 uint16_t uCsStart;
9044
9045 /** Whether we've actually modified the 1st execution control field. */
9046 bool fModifiedProcCtls : 1;
9047 /** Whether we've actually modified the 2nd execution control field. */
9048 bool fModifiedProcCtls2 : 1;
9049 /** Whether we've actually modified the exception bitmap. */
9050 bool fModifiedXcptBitmap : 1;
9051
9052 /** We desire the modified the CR0 mask to be cleared. */
9053 bool fClearCr0Mask : 1;
9054 /** We desire the modified the CR4 mask to be cleared. */
9055 bool fClearCr4Mask : 1;
9056 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9057 uint32_t fCpe1Extra;
9058 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9059 uint32_t fCpe1Unwanted;
9060 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9061 uint32_t fCpe2Extra;
9062 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9063 uint32_t bmXcptExtra;
9064 /** The sequence number of the Dtrace provider settings the state was
9065 * configured against. */
9066 uint32_t uDtraceSettingsSeqNo;
9067 /** VM-exits to check (one bit per VM-exit). */
9068 uint32_t bmExitsToCheck[3];
9069
9070 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9071 uint32_t fProcCtlsInitial;
9072 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9073 uint32_t fProcCtls2Initial;
9074 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9075 uint32_t bmXcptInitial;
9076} VMXRUNDBGSTATE;
9077AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9078typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9079
9080
9081/**
9082 * Initializes the VMXRUNDBGSTATE structure.
9083 *
9084 * @param pVCpu The cross context virtual CPU structure of the
9085 * calling EMT.
9086 * @param pDbgState The structure to initialize.
9087 */
9088static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9089{
9090 pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
9091 pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
9092
9093 pDbgState->fModifiedProcCtls = false;
9094 pDbgState->fModifiedProcCtls2 = false;
9095 pDbgState->fModifiedXcptBitmap = false;
9096 pDbgState->fClearCr0Mask = false;
9097 pDbgState->fClearCr4Mask = false;
9098 pDbgState->fCpe1Extra = 0;
9099 pDbgState->fCpe1Unwanted = 0;
9100 pDbgState->fCpe2Extra = 0;
9101 pDbgState->bmXcptExtra = 0;
9102 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9103 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9104 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9105}
9106
9107
9108/**
9109 * Updates the VMSC fields with changes requested by @a pDbgState.
9110 *
9111 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9112 * immediately before executing guest code, i.e. when interrupts are disabled.
9113 * We don't check status codes here as we cannot easily assert or return in the
9114 * latter case.
9115 *
9116 * @param pVCpu The cross context virtual CPU structure.
9117 * @param pDbgState The debug state.
9118 */
9119static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9120{
9121 /*
9122 * Ensure desired flags in VMCS control fields are set.
9123 * (Ignoring write failure here, as we're committed and it's just debug extras.)
9124 *
9125 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9126 * there should be no stale data in pCtx at this point.
9127 */
9128 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9129 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9130 {
9131 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
9132 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9133 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9134 Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9135 pDbgState->fModifiedProcCtls = true;
9136 }
9137
9138 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9139 {
9140 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9141 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9142 Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9143 pDbgState->fModifiedProcCtls2 = true;
9144 }
9145
9146 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9147 {
9148 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9149 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9150 Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9151 pDbgState->fModifiedXcptBitmap = true;
9152 }
9153
9154 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32Cr0Mask != 0)
9155 {
9156 pVCpu->hm.s.vmx.u32Cr0Mask = 0;
9157 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9158 Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9159 }
9160
9161 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32Cr4Mask != 0)
9162 {
9163 pVCpu->hm.s.vmx.u32Cr4Mask = 0;
9164 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9165 Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9166 }
9167}
9168
9169
9170/**
9171 * Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
9172 * re-entry next time around.
9173 *
9174 * @returns Strict VBox status code (i.e. informational status codes too).
9175 * @param pVCpu The cross context virtual CPU structure.
9176 * @param pDbgState The debug state.
9177 * @param rcStrict The return code from executing the guest using single
9178 * stepping.
9179 */
9180static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9181{
9182 /*
9183 * Restore VM-exit control settings as we may not reenter this function the
9184 * next time around.
9185 */
9186 /* We reload the initial value, trigger what we can of recalculations the
9187 next time around. From the looks of things, that's all that's required atm. */
9188 if (pDbgState->fModifiedProcCtls)
9189 {
9190 if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9191 pDbgState->fProcCtlsInitial |= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9192 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9193 AssertRCReturn(rc2, rc2);
9194 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9195 }
9196
9197 /* We're currently the only ones messing with this one, so just restore the
9198 cached value and reload the field. */
9199 if ( pDbgState->fModifiedProcCtls2
9200 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9201 {
9202 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9203 AssertRCReturn(rc2, rc2);
9204 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9205 }
9206
9207 /* If we've modified the exception bitmap, we restore it and trigger
9208 reloading and partial recalculation the next time around. */
9209 if (pDbgState->fModifiedXcptBitmap)
9210 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9211
9212 return rcStrict;
9213}
9214
9215
9216/**
9217 * Configures VM-exit controls for current DBGF and DTrace settings.
9218 *
9219 * This updates @a pDbgState and the VMCS execution control fields to reflect
9220 * the necessary VM-exits demanded by DBGF and DTrace.
9221 *
9222 * @param pVCpu The cross context virtual CPU structure.
9223 * @param pDbgState The debug state.
9224 * @param pVmxTransient Pointer to the VMX transient structure. May update
9225 * fUpdateTscOffsettingAndPreemptTimer.
9226 */
9227static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9228{
9229 /*
9230 * Take down the dtrace serial number so we can spot changes.
9231 */
9232 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9233 ASMCompilerBarrier();
9234
9235 /*
9236 * We'll rebuild most of the middle block of data members (holding the
9237 * current settings) as we go along here, so start by clearing it all.
9238 */
9239 pDbgState->bmXcptExtra = 0;
9240 pDbgState->fCpe1Extra = 0;
9241 pDbgState->fCpe1Unwanted = 0;
9242 pDbgState->fCpe2Extra = 0;
9243 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9244 pDbgState->bmExitsToCheck[i] = 0;
9245
9246 /*
9247 * Software interrupts (INT XXh) - no idea how to trigger these...
9248 */
9249 PVM pVM = pVCpu->CTX_SUFF(pVM);
9250 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9251 || VBOXVMM_INT_SOFTWARE_ENABLED())
9252 {
9253 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9254 }
9255
9256 /*
9257 * INT3 breakpoints - triggered by #BP exceptions.
9258 */
9259 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9260 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9261
9262 /*
9263 * Exception bitmap and XCPT events+probes.
9264 */
9265 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9266 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9267 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9268
9269 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9270 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9271 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9272 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9273 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9274 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9275 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9276 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9277 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9278 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9279 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9280 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9281 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9282 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9283 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9284 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9285 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9286 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9287
9288 if (pDbgState->bmXcptExtra)
9289 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9290
9291 /*
9292 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9293 *
9294 * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
9295 * So, when adding/changing/removing please don't forget to update it.
9296 *
9297 * Some of the macros are picking up local variables to save horizontal space,
9298 * (being able to see it in a table is the lesser evil here).
9299 */
9300#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9301 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9302 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9303#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9304 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9305 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9306 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9307 } else do { } while (0)
9308#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9309 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9310 { \
9311 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9312 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9313 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9314 } else do { } while (0)
9315#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9316 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9317 { \
9318 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9319 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9320 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9321 } else do { } while (0)
9322#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9323 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9324 { \
9325 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9326 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9327 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9328 } else do { } while (0)
9329
9330 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9331 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9332 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9333 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9334 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9335
9336 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9337 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9338 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9339 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9340 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
9341 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9342 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9343 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9344 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
9345 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9346 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
9347 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9348 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
9349 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9350 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9351 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9352 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9353 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9354 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9355 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9356 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9357 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9358 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9359 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9360 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9361 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9362 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9363 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9364 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9365 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9366 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9367 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9368 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9369 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9370 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9371 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9372
9373 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9374 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9375 {
9376 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_APIC_TPR);
9377 AssertRC(rc);
9378
9379#if 0 /** @todo fix me */
9380 pDbgState->fClearCr0Mask = true;
9381 pDbgState->fClearCr4Mask = true;
9382#endif
9383 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9384 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_STORE_EXIT | VMX_PROC_CTLS_CR8_STORE_EXIT;
9385 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9386 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_LOAD_EXIT | VMX_PROC_CTLS_CR8_LOAD_EXIT;
9387 pDbgState->fCpe1Unwanted |= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
9388 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9389 require clearing here and in the loop if we start using it. */
9390 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9391 }
9392 else
9393 {
9394 if (pDbgState->fClearCr0Mask)
9395 {
9396 pDbgState->fClearCr0Mask = false;
9397 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9398 }
9399 if (pDbgState->fClearCr4Mask)
9400 {
9401 pDbgState->fClearCr4Mask = false;
9402 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9403 }
9404 }
9405 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9406 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9407
9408 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9409 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9410 {
9411 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9412 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9413 }
9414 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9415 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9416
9417 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
9418 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9419 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
9420 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9421 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
9422 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9423 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
9424 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9425#if 0 /** @todo too slow, fix handler. */
9426 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
9427#endif
9428 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9429
9430 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9431 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9432 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9433 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9434 {
9435 pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
9436 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
9437 }
9438 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9439 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9440 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9441 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9442
9443 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9444 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9445 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9446 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9447 {
9448 pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
9449 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
9450 }
9451 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
9452 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
9453 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
9454 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
9455
9456 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9457 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9458 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
9459 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9460 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9461 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9462 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
9463 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9464 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9465 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9466 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
9467 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9468 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
9469 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9470 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9471 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9472 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
9473 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9474 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9475 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9476 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9477 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9478
9479#undef IS_EITHER_ENABLED
9480#undef SET_ONLY_XBM_IF_EITHER_EN
9481#undef SET_CPE1_XBM_IF_EITHER_EN
9482#undef SET_CPEU_XBM_IF_EITHER_EN
9483#undef SET_CPE2_XBM_IF_EITHER_EN
9484
9485 /*
9486 * Sanitize the control stuff.
9487 */
9488 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
9489 if (pDbgState->fCpe2Extra)
9490 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
9491 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
9492 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0;
9493 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
9494 {
9495 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9496 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9497 }
9498
9499 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9500 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9501 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9502 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9503}
9504
9505
9506/**
9507 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9508 * appropriate.
9509 *
9510 * The caller has checked the VM-exit against the
9511 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9512 * already, so we don't have to do that either.
9513 *
9514 * @returns Strict VBox status code (i.e. informational status codes too).
9515 * @param pVCpu The cross context virtual CPU structure.
9516 * @param pVmxTransient Pointer to the VMX-transient structure.
9517 * @param uExitReason The VM-exit reason.
9518 *
9519 * @remarks The name of this function is displayed by dtrace, so keep it short
9520 * and to the point. No longer than 33 chars long, please.
9521 */
9522static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9523{
9524 /*
9525 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9526 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9527 *
9528 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9529 * does. Must add/change/remove both places. Same ordering, please.
9530 *
9531 * Added/removed events must also be reflected in the next section
9532 * where we dispatch dtrace events.
9533 */
9534 bool fDtrace1 = false;
9535 bool fDtrace2 = false;
9536 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9537 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9538 uint32_t uEventArg = 0;
9539#define SET_EXIT(a_EventSubName) \
9540 do { \
9541 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9542 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9543 } while (0)
9544#define SET_BOTH(a_EventSubName) \
9545 do { \
9546 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9547 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9548 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9549 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9550 } while (0)
9551 switch (uExitReason)
9552 {
9553 case VMX_EXIT_MTF:
9554 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9555
9556 case VMX_EXIT_XCPT_OR_NMI:
9557 {
9558 uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9559 switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
9560 {
9561 case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
9562 case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
9563 case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
9564 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9565 {
9566 if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
9567 {
9568 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9569 uEventArg = pVmxTransient->uExitIntErrorCode;
9570 }
9571 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9572 switch (enmEvent1)
9573 {
9574 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9575 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9576 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9577 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9578 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9579 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9580 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9581 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9582 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9583 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9584 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9585 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9586 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9587 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9588 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9589 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9590 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9591 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9592 default: break;
9593 }
9594 }
9595 else
9596 AssertFailed();
9597 break;
9598
9599 case VMX_EXIT_INT_INFO_TYPE_SW_INT:
9600 uEventArg = idxVector;
9601 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9602 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9603 break;
9604 }
9605 break;
9606 }
9607
9608 case VMX_EXIT_TRIPLE_FAULT:
9609 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9610 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9611 break;
9612 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9613 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9614 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9615 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9616 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9617
9618 /* Instruction specific VM-exits: */
9619 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9620 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9621 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9622 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9623 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9624 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9625 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9626 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9627 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9628 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9629 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9630 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9631 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9632 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9633 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9634 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9635 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9636 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9637 case VMX_EXIT_MOV_CRX:
9638 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9639 if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9640 SET_BOTH(CRX_READ);
9641 else
9642 SET_BOTH(CRX_WRITE);
9643 uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
9644 break;
9645 case VMX_EXIT_MOV_DRX:
9646 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9647 if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
9648 == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9649 SET_BOTH(DRX_READ);
9650 else
9651 SET_BOTH(DRX_WRITE);
9652 uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
9653 break;
9654 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9655 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9656 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9657 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9658 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9659 case VMX_EXIT_GDTR_IDTR_ACCESS:
9660 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9661 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
9662 {
9663 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9664 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9665 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9666 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9667 }
9668 break;
9669
9670 case VMX_EXIT_LDTR_TR_ACCESS:
9671 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9672 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
9673 {
9674 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9675 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9676 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9677 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9678 }
9679 break;
9680
9681 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9682 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9683 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9684 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9685 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9686 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9687 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9688 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9689 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9690 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9691 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9692
9693 /* Events that aren't relevant at this point. */
9694 case VMX_EXIT_EXT_INT:
9695 case VMX_EXIT_INT_WINDOW:
9696 case VMX_EXIT_NMI_WINDOW:
9697 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9698 case VMX_EXIT_PREEMPT_TIMER:
9699 case VMX_EXIT_IO_INSTR:
9700 break;
9701
9702 /* Errors and unexpected events. */
9703 case VMX_EXIT_INIT_SIGNAL:
9704 case VMX_EXIT_SIPI:
9705 case VMX_EXIT_IO_SMI:
9706 case VMX_EXIT_SMI:
9707 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9708 case VMX_EXIT_ERR_MSR_LOAD:
9709 case VMX_EXIT_ERR_MACHINE_CHECK:
9710 break;
9711
9712 default:
9713 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9714 break;
9715 }
9716#undef SET_BOTH
9717#undef SET_EXIT
9718
9719 /*
9720 * Dtrace tracepoints go first. We do them here at once so we don't
9721 * have to copy the guest state saving and stuff a few dozen times.
9722 * Down side is that we've got to repeat the switch, though this time
9723 * we use enmEvent since the probes are a subset of what DBGF does.
9724 */
9725 if (fDtrace1 || fDtrace2)
9726 {
9727 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9728 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9729 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9730 switch (enmEvent1)
9731 {
9732 /** @todo consider which extra parameters would be helpful for each probe. */
9733 case DBGFEVENT_END: break;
9734 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
9735 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
9736 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
9737 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
9738 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
9739 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
9740 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
9741 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
9742 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
9743 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
9744 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
9745 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
9746 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
9747 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
9748 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
9749 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
9750 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
9751 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
9752 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9753 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9754 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
9755 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
9756 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
9757 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
9758 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
9759 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
9760 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
9761 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9762 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9763 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9764 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9765 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9766 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
9767 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9768 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
9769 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
9770 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
9771 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
9772 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
9773 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
9774 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
9775 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
9776 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
9777 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
9778 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
9779 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
9780 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
9781 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
9782 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
9783 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
9784 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
9785 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
9786 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
9787 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
9788 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
9789 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
9790 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
9791 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
9792 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
9793 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
9794 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
9795 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
9796 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
9797 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
9798 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
9799 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
9800 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9801 }
9802 switch (enmEvent2)
9803 {
9804 /** @todo consider which extra parameters would be helpful for each probe. */
9805 case DBGFEVENT_END: break;
9806 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
9807 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9808 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
9809 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
9810 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
9811 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
9812 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
9813 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
9814 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
9815 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9816 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9817 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9818 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9819 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9820 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
9821 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9822 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
9823 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
9824 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
9825 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
9826 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
9827 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
9828 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
9829 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
9830 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
9831 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
9832 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
9833 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
9834 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
9835 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
9836 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
9837 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
9838 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
9839 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
9840 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
9841 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
9842 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
9843 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
9844 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
9845 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
9846 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
9847 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
9848 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
9849 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
9850 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
9851 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
9852 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
9853 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
9854 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
9855 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
9856 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
9857 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
9858 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9859 }
9860 }
9861
9862 /*
9863 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9864 * the DBGF call will do a full check).
9865 *
9866 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9867 * Note! If we have to events, we prioritize the first, i.e. the instruction
9868 * one, in order to avoid event nesting.
9869 */
9870 PVM pVM = pVCpu->CTX_SUFF(pVM);
9871 if ( enmEvent1 != DBGFEVENT_END
9872 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9873 {
9874 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
9875 if (rcStrict != VINF_SUCCESS)
9876 return rcStrict;
9877 }
9878 else if ( enmEvent2 != DBGFEVENT_END
9879 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9880 {
9881 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
9882 if (rcStrict != VINF_SUCCESS)
9883 return rcStrict;
9884 }
9885
9886 return VINF_SUCCESS;
9887}
9888
9889
9890/**
9891 * Single-stepping VM-exit filtering.
9892 *
9893 * This is preprocessing the VM-exits and deciding whether we've gotten far
9894 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9895 * handling is performed.
9896 *
9897 * @returns Strict VBox status code (i.e. informational status codes too).
9898 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9899 * @param pVmxTransient Pointer to the VMX-transient structure.
9900 * @param pDbgState The debug state.
9901 */
9902DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
9903{
9904 /*
9905 * Expensive (saves context) generic dtrace VM-exit probe.
9906 */
9907 uint32_t const uExitReason = pVmxTransient->uExitReason;
9908 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9909 { /* more likely */ }
9910 else
9911 {
9912 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9913 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9914 AssertRC(rc);
9915 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
9916 }
9917
9918 /*
9919 * Check for host NMI, just to get that out of the way.
9920 */
9921 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9922 { /* normally likely */ }
9923 else
9924 {
9925 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9926 AssertRCReturn(rc2, rc2);
9927 uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
9928 if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
9929 return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
9930 }
9931
9932 /*
9933 * Check for single stepping event if we're stepping.
9934 */
9935 if (pVCpu->hm.s.fSingleInstruction)
9936 {
9937 switch (uExitReason)
9938 {
9939 case VMX_EXIT_MTF:
9940 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9941
9942 /* Various events: */
9943 case VMX_EXIT_XCPT_OR_NMI:
9944 case VMX_EXIT_EXT_INT:
9945 case VMX_EXIT_TRIPLE_FAULT:
9946 case VMX_EXIT_INT_WINDOW:
9947 case VMX_EXIT_NMI_WINDOW:
9948 case VMX_EXIT_TASK_SWITCH:
9949 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9950 case VMX_EXIT_APIC_ACCESS:
9951 case VMX_EXIT_EPT_VIOLATION:
9952 case VMX_EXIT_EPT_MISCONFIG:
9953 case VMX_EXIT_PREEMPT_TIMER:
9954
9955 /* Instruction specific VM-exits: */
9956 case VMX_EXIT_CPUID:
9957 case VMX_EXIT_GETSEC:
9958 case VMX_EXIT_HLT:
9959 case VMX_EXIT_INVD:
9960 case VMX_EXIT_INVLPG:
9961 case VMX_EXIT_RDPMC:
9962 case VMX_EXIT_RDTSC:
9963 case VMX_EXIT_RSM:
9964 case VMX_EXIT_VMCALL:
9965 case VMX_EXIT_VMCLEAR:
9966 case VMX_EXIT_VMLAUNCH:
9967 case VMX_EXIT_VMPTRLD:
9968 case VMX_EXIT_VMPTRST:
9969 case VMX_EXIT_VMREAD:
9970 case VMX_EXIT_VMRESUME:
9971 case VMX_EXIT_VMWRITE:
9972 case VMX_EXIT_VMXOFF:
9973 case VMX_EXIT_VMXON:
9974 case VMX_EXIT_MOV_CRX:
9975 case VMX_EXIT_MOV_DRX:
9976 case VMX_EXIT_IO_INSTR:
9977 case VMX_EXIT_RDMSR:
9978 case VMX_EXIT_WRMSR:
9979 case VMX_EXIT_MWAIT:
9980 case VMX_EXIT_MONITOR:
9981 case VMX_EXIT_PAUSE:
9982 case VMX_EXIT_GDTR_IDTR_ACCESS:
9983 case VMX_EXIT_LDTR_TR_ACCESS:
9984 case VMX_EXIT_INVEPT:
9985 case VMX_EXIT_RDTSCP:
9986 case VMX_EXIT_INVVPID:
9987 case VMX_EXIT_WBINVD:
9988 case VMX_EXIT_XSETBV:
9989 case VMX_EXIT_RDRAND:
9990 case VMX_EXIT_INVPCID:
9991 case VMX_EXIT_VMFUNC:
9992 case VMX_EXIT_RDSEED:
9993 case VMX_EXIT_XSAVES:
9994 case VMX_EXIT_XRSTORS:
9995 {
9996 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
9997 AssertRCReturn(rc, rc);
9998 if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
9999 || pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
10000 return VINF_EM_DBG_STEPPED;
10001 break;
10002 }
10003
10004 /* Errors and unexpected events: */
10005 case VMX_EXIT_INIT_SIGNAL:
10006 case VMX_EXIT_SIPI:
10007 case VMX_EXIT_IO_SMI:
10008 case VMX_EXIT_SMI:
10009 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10010 case VMX_EXIT_ERR_MSR_LOAD:
10011 case VMX_EXIT_ERR_MACHINE_CHECK:
10012 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
10013 break;
10014
10015 default:
10016 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10017 break;
10018 }
10019 }
10020
10021 /*
10022 * Check for debugger event breakpoints and dtrace probes.
10023 */
10024 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10025 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10026 {
10027 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
10028 if (rcStrict != VINF_SUCCESS)
10029 return rcStrict;
10030 }
10031
10032 /*
10033 * Normal processing.
10034 */
10035#ifdef HMVMX_USE_FUNCTION_TABLE
10036 return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
10037#else
10038 return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
10039#endif
10040}
10041
10042
10043/**
10044 * Single steps guest code using VT-x.
10045 *
10046 * @returns Strict VBox status code (i.e. informational status codes too).
10047 * @param pVCpu The cross context virtual CPU structure.
10048 *
10049 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10050 */
10051static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu)
10052{
10053 VMXTRANSIENT VmxTransient;
10054 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10055
10056 /* Set HMCPU indicators. */
10057 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10058 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
10059 pVCpu->hm.s.fDebugWantRdTscExit = false;
10060 pVCpu->hm.s.fUsingDebugLoop = true;
10061
10062 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10063 VMXRUNDBGSTATE DbgState;
10064 hmR0VmxRunDebugStateInit(pVCpu, &DbgState);
10065 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
10066
10067 /*
10068 * The loop.
10069 */
10070 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10071 for (uint32_t cLoops = 0; ; cLoops++)
10072 {
10073 Assert(!HMR0SuspendPending());
10074 HMVMX_ASSERT_CPU_SAFE(pVCpu);
10075 bool fStepping = pVCpu->hm.s.fSingleInstruction;
10076
10077 /*
10078 * Preparatory work for running guest code, this may force us to return
10079 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10080 */
10081 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10082 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10083 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
10084 if (rcStrict != VINF_SUCCESS)
10085 break;
10086
10087 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10088 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10089
10090 /*
10091 * Now we can run the guest code.
10092 */
10093 int rcRun = hmR0VmxRunGuest(pVCpu);
10094
10095 /*
10096 * Restore any residual host-state and save any bits shared between host
10097 * and guest into the guest-CPU state. Re-enables interrupts!
10098 */
10099 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10100
10101 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10102 if (RT_SUCCESS(rcRun))
10103 { /* very likely */ }
10104 else
10105 {
10106 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10107 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10108 return rcRun;
10109 }
10110
10111 /* Profile the VM-exit. */
10112 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10113 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10114 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10115 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10116 HMVMX_START_EXIT_DISPATCH_PROF();
10117
10118 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10119
10120 /*
10121 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10122 */
10123 rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
10124 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10125 if (rcStrict != VINF_SUCCESS)
10126 break;
10127 if (cLoops > pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
10128 {
10129 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10130 rcStrict = VINF_EM_RAW_INTERRUPT;
10131 break;
10132 }
10133
10134 /*
10135 * Stepping: Did the RIP change, if so, consider it a single step.
10136 * Otherwise, make sure one of the TFs gets set.
10137 */
10138 if (fStepping)
10139 {
10140 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
10141 AssertRC(rc);
10142 if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
10143 || pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
10144 {
10145 rcStrict = VINF_EM_DBG_STEPPED;
10146 break;
10147 }
10148 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
10149 }
10150
10151 /*
10152 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10153 */
10154 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10155 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
10156 }
10157
10158 /*
10159 * Clear the X86_EFL_TF if necessary.
10160 */
10161 if (pVCpu->hm.s.fClearTrapFlag)
10162 {
10163 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
10164 AssertRC(rc);
10165 pVCpu->hm.s.fClearTrapFlag = false;
10166 pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
10167 }
10168 /** @todo there seems to be issues with the resume flag when the monitor trap
10169 * flag is pending without being used. Seen early in bios init when
10170 * accessing APIC page in protected mode. */
10171
10172 /*
10173 * Restore VM-exit control settings as we may not reenter this function the
10174 * next time around.
10175 */
10176 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10177
10178 /* Restore HMCPU indicators. */
10179 pVCpu->hm.s.fUsingDebugLoop = false;
10180 pVCpu->hm.s.fDebugWantRdTscExit = false;
10181 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10182
10183 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10184 return rcStrict;
10185}
10186
10187
10188/** @} */
10189
10190
10191/**
10192 * Checks if any expensive dtrace probes are enabled and we should go to the
10193 * debug loop.
10194 *
10195 * @returns true if we should use debug loop, false if not.
10196 */
10197static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10198{
10199 /* It's probably faster to OR the raw 32-bit counter variables together.
10200 Since the variables are in an array and the probes are next to one
10201 another (more or less), we have good locality. So, better read
10202 eight-nine cache lines ever time and only have one conditional, than
10203 128+ conditionals, right? */
10204 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10205 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10206 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10207 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10208 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10209 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10210 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10211 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10212 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10213 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10214 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10215 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10216 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10217 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10218 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10219 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10220 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10221 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10222 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10223 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10224 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10225 ) != 0
10226 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10227 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10228 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10229 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10230 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10231 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10232 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10233 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10234 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10235 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10236 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10237 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10238 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10239 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10240 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10241 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10242 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10243 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10244 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10245 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10246 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10247 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10248 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10249 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10250 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10251 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10252 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10253 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10254 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10255 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10256 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10257 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10258 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10259 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10260 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10261 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10262 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10263 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10264 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10265 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10266 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10267 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10268 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10269 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10270 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10271 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10272 ) != 0
10273 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10274 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10275 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10276 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10277 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10278 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10279 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10280 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10281 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10282 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10283 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10284 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10285 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10286 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10287 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10288 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10289 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10290 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10291 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10292 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10293 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10294 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10295 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10296 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10297 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10298 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10299 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10300 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10301 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10302 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10303 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10304 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10305 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10306 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10307 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10308 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10309 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10310 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10311 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10312 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10313 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10314 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10315 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10316 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10317 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10318 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10319 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10320 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10321 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10322 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10323 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10324 ) != 0;
10325}
10326
10327
10328/**
10329 * Runs the guest code using VT-x.
10330 *
10331 * @returns Strict VBox status code (i.e. informational status codes too).
10332 * @param pVCpu The cross context virtual CPU structure.
10333 */
10334VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
10335{
10336 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10337 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10338 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10339 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
10340
10341 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10342
10343 VBOXSTRICTRC rcStrict;
10344 if ( !pVCpu->hm.s.fUseDebugLoop
10345 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10346 && !DBGFIsStepping(pVCpu)
10347 && !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
10348 rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu);
10349 else
10350 rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu);
10351
10352 if (rcStrict == VERR_EM_INTERPRETER)
10353 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10354 else if (rcStrict == VINF_EM_RESET)
10355 rcStrict = VINF_EM_TRIPLE_FAULT;
10356
10357 int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
10358 if (RT_FAILURE(rc2))
10359 {
10360 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10361 rcStrict = rc2;
10362 }
10363 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10364 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10365 return rcStrict;
10366}
10367
10368
10369#ifndef HMVMX_USE_FUNCTION_TABLE
10370DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10371{
10372#ifdef DEBUG_ramshankar
10373#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10374 do { \
10375 if (a_fSave != 0) \
10376 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10377 VBOXSTRICTRC rcStrict = a_CallExpr; \
10378 if (a_fSave != 0) \
10379 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10380 return rcStrict; \
10381 } while (0)
10382#else
10383# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10384#endif
10385 switch (rcReason)
10386 {
10387 case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
10388 case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
10389 case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
10390 case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
10391 case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
10392 case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
10393 case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
10394 case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
10395 case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
10396 case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
10397 case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
10398 case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
10399 case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
10400 case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
10401 case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
10402 case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
10403 case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
10404 case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
10405 case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
10406 case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
10407 case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
10408 case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
10409 case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
10410 case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pVmxTransient));
10411 case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
10412 case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
10413 case VMX_EXIT_GDTR_IDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10414 case VMX_EXIT_LDTR_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10415 case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
10416 case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
10417 case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
10418 case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
10419 case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
10420 case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
10421
10422 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
10423 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
10424 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pVmxTransient);
10425 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pVmxTransient);
10426 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pVmxTransient);
10427 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pVmxTransient);
10428 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pVmxTransient);
10429 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
10430 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pVmxTransient);
10431
10432 case VMX_EXIT_VMCLEAR:
10433 case VMX_EXIT_VMLAUNCH:
10434 case VMX_EXIT_VMPTRLD:
10435 case VMX_EXIT_VMPTRST:
10436 case VMX_EXIT_VMREAD:
10437 case VMX_EXIT_VMRESUME:
10438 case VMX_EXIT_VMWRITE:
10439 case VMX_EXIT_VMXOFF:
10440 case VMX_EXIT_VMXON:
10441 case VMX_EXIT_INVEPT:
10442 case VMX_EXIT_INVVPID:
10443 case VMX_EXIT_VMFUNC:
10444 case VMX_EXIT_XSAVES:
10445 case VMX_EXIT_XRSTORS:
10446 return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
10447
10448 case VMX_EXIT_ENCLS:
10449 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10450 case VMX_EXIT_PML_FULL:
10451 default:
10452 return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
10453 }
10454#undef VMEXIT_CALL_RET
10455}
10456#endif /* !HMVMX_USE_FUNCTION_TABLE */
10457
10458
10459#ifdef VBOX_STRICT
10460/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10461# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10462 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10463
10464# define HMVMX_ASSERT_PREEMPT_CPUID() \
10465 do { \
10466 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10467 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10468 } while (0)
10469
10470# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10471 do { \
10472 AssertPtr((a_pVCpu)); \
10473 AssertPtr((a_pVmxTransient)); \
10474 Assert((a_pVmxTransient)->fVMEntryFailed == false); \
10475 Assert(ASMIntAreEnabled()); \
10476 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10477 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10478 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
10479 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10480 if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
10481 HMVMX_ASSERT_PREEMPT_CPUID(); \
10482 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10483 } while (0)
10484
10485# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10486 do { \
10487 Log4Func(("\n")); \
10488 } while (0)
10489#else
10490# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10491 do { \
10492 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10493 NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
10494 } while (0)
10495# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
10496#endif
10497
10498
10499/**
10500 * Advances the guest RIP by the specified number of bytes.
10501 *
10502 * @param pVCpu The cross context virtual CPU structure.
10503 * @param cbInstr Number of bytes to advance the RIP by.
10504 *
10505 * @remarks No-long-jump zone!!!
10506 */
10507DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
10508{
10509 /* Advance the RIP. */
10510 pVCpu->cpum.GstCtx.rip += cbInstr;
10511 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10512
10513 /* Update interrupt inhibition. */
10514 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10515 && pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
10516 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10517}
10518
10519
10520/**
10521 * Advances the guest RIP after reading it from the VMCS.
10522 *
10523 * @returns VBox status code, no informational status codes.
10524 * @param pVCpu The cross context virtual CPU structure.
10525 * @param pVmxTransient Pointer to the VMX transient structure.
10526 *
10527 * @remarks No-long-jump zone!!!
10528 */
10529static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10530{
10531 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10532 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS);
10533 AssertRCReturn(rc, rc);
10534
10535 hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
10536
10537 /*
10538 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10539 * pending debug exception field as it takes care of priority of events.
10540 *
10541 * See Intel spec. 32.2.1 "Debug Exceptions".
10542 */
10543 if ( !pVCpu->hm.s.fSingleInstruction
10544 && pVCpu->cpum.GstCtx.eflags.Bits.u1TF)
10545 {
10546 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10547 AssertRCReturn(rc, rc);
10548 }
10549
10550 return VINF_SUCCESS;
10551}
10552
10553
10554/**
10555 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10556 * and update error record fields accordingly.
10557 *
10558 * @return VMX_IGS_* return codes.
10559 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10560 * wrong with the guest state.
10561 *
10562 * @param pVCpu The cross context virtual CPU structure.
10563 *
10564 * @remarks This function assumes our cache of the VMCS controls
10565 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10566 */
10567static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu)
10568{
10569#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10570#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10571 uError = (err); \
10572 break; \
10573 } else do { } while (0)
10574
10575 int rc;
10576 PVM pVM = pVCpu->CTX_SUFF(pVM);
10577 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10578 uint32_t uError = VMX_IGS_ERROR;
10579 uint32_t u32Val;
10580 bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10581
10582 do
10583 {
10584 /*
10585 * CR0.
10586 */
10587 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10588 uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10589 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10590 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10591 if (fUnrestrictedGuest)
10592 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
10593
10594 uint32_t u32GuestCr0;
10595 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10596 AssertRCBreak(rc);
10597 HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10598 HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10599 if ( !fUnrestrictedGuest
10600 && (u32GuestCr0 & X86_CR0_PG)
10601 && !(u32GuestCr0 & X86_CR0_PE))
10602 {
10603 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10604 }
10605
10606 /*
10607 * CR4.
10608 */
10609 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10610 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10611
10612 uint32_t u32GuestCr4;
10613 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10614 AssertRCBreak(rc);
10615 HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10616 HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10617
10618 /*
10619 * IA32_DEBUGCTL MSR.
10620 */
10621 uint64_t u64Val;
10622 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10623 AssertRCBreak(rc);
10624 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10625 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10626 {
10627 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10628 }
10629 uint64_t u64DebugCtlMsr = u64Val;
10630
10631#ifdef VBOX_STRICT
10632 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10633 AssertRCBreak(rc);
10634 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10635#endif
10636 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
10637
10638 /*
10639 * RIP and RFLAGS.
10640 */
10641 uint32_t u32Eflags;
10642#if HC_ARCH_BITS == 64
10643 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10644 AssertRCBreak(rc);
10645 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10646 if ( !fLongModeGuest
10647 || !pCtx->cs.Attr.n.u1Long)
10648 {
10649 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10650 }
10651 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10652 * must be identical if the "IA-32e mode guest" VM-entry
10653 * control is 1 and CS.L is 1. No check applies if the
10654 * CPU supports 64 linear-address bits. */
10655
10656 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10657 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10658 AssertRCBreak(rc);
10659 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10660 VMX_IGS_RFLAGS_RESERVED);
10661 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10662 u32Eflags = u64Val;
10663#else
10664 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10665 AssertRCBreak(rc);
10666 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10667 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10668#endif
10669
10670 if ( fLongModeGuest
10671 || ( fUnrestrictedGuest
10672 && !(u32GuestCr0 & X86_CR0_PE)))
10673 {
10674 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10675 }
10676
10677 uint32_t u32EntryInfo;
10678 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10679 AssertRCBreak(rc);
10680 if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
10681 && VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
10682 {
10683 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10684 }
10685
10686 /*
10687 * 64-bit checks.
10688 */
10689#if HC_ARCH_BITS == 64
10690 if (fLongModeGuest)
10691 {
10692 HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10693 HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10694 }
10695
10696 if ( !fLongModeGuest
10697 && (u32GuestCr4 & X86_CR4_PCIDE))
10698 {
10699 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10700 }
10701
10702 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10703 * 51:32 beyond the processor's physical-address width are 0. */
10704
10705 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10706 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10707 {
10708 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10709 }
10710
10711 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10712 AssertRCBreak(rc);
10713 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10714
10715 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10716 AssertRCBreak(rc);
10717 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10718#endif
10719
10720 /*
10721 * PERF_GLOBAL MSR.
10722 */
10723 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
10724 {
10725 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10726 AssertRCBreak(rc);
10727 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10728 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10729 }
10730
10731 /*
10732 * PAT MSR.
10733 */
10734 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
10735 {
10736 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10737 AssertRCBreak(rc);
10738 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10739 for (unsigned i = 0; i < 8; i++)
10740 {
10741 uint8_t u8Val = (u64Val & 0xff);
10742 if ( u8Val != 0 /* UC */
10743 && u8Val != 1 /* WC */
10744 && u8Val != 4 /* WT */
10745 && u8Val != 5 /* WP */
10746 && u8Val != 6 /* WB */
10747 && u8Val != 7 /* UC- */)
10748 {
10749 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10750 }
10751 u64Val >>= 8;
10752 }
10753 }
10754
10755 /*
10756 * EFER MSR.
10757 */
10758 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
10759 {
10760 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10761 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10762 AssertRCBreak(rc);
10763 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10764 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10765 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10766 & VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
10767 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10768 /** @todo r=ramshankar: Unrestricted check here is probably wrong, see
10769 * iemVmxVmentryCheckGuestState(). */
10770 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10771 || !(u32GuestCr0 & X86_CR0_PG)
10772 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10773 VMX_IGS_EFER_LMA_LME_MISMATCH);
10774 }
10775
10776 /*
10777 * Segment registers.
10778 */
10779 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10780 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10781 if (!(u32Eflags & X86_EFL_VM))
10782 {
10783 /* CS */
10784 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10785 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10786 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10787 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10788 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10789 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10790 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10791 /* CS cannot be loaded with NULL in protected mode. */
10792 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10793 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10794 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10795 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10796 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10797 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10798 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10799 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10800 else
10801 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10802
10803 /* SS */
10804 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10805 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10806 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10807 if ( !(pCtx->cr0 & X86_CR0_PE)
10808 || pCtx->cs.Attr.n.u4Type == 3)
10809 {
10810 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10811 }
10812 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10813 {
10814 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10815 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10816 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10817 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10818 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10819 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10820 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10821 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10822 }
10823
10824 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmenReg(). */
10825 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10826 {
10827 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10828 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10829 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10830 || pCtx->ds.Attr.n.u4Type > 11
10831 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10832 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10833 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10834 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10835 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10836 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10837 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10838 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10839 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10840 }
10841 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10842 {
10843 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10844 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10845 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10846 || pCtx->es.Attr.n.u4Type > 11
10847 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10848 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10849 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10850 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10851 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10852 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10853 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10854 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10855 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10856 }
10857 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10858 {
10859 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10860 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10861 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10862 || pCtx->fs.Attr.n.u4Type > 11
10863 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10864 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10865 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10866 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10867 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10868 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10869 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10870 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10871 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10872 }
10873 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10874 {
10875 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10876 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10877 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10878 || pCtx->gs.Attr.n.u4Type > 11
10879 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10880 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10881 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10882 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10883 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10884 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10885 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10886 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10887 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10888 }
10889 /* 64-bit capable CPUs. */
10890#if HC_ARCH_BITS == 64
10891 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10892 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10893 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10894 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10895 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10896 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10897 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10898 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10899 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10900 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10901 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10902#endif
10903 }
10904 else
10905 {
10906 /* V86 mode checks. */
10907 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10908 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10909 {
10910 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10911 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10912 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10913 }
10914 else
10915 {
10916 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10917 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10918 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10919 }
10920
10921 /* CS */
10922 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10923 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10924 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10925 /* SS */
10926 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10927 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10928 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10929 /* DS */
10930 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10931 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10932 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10933 /* ES */
10934 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10935 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10936 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10937 /* FS */
10938 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10939 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10940 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10941 /* GS */
10942 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10943 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10944 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10945 /* 64-bit capable CPUs. */
10946#if HC_ARCH_BITS == 64
10947 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10948 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10949 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10950 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10951 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10952 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10953 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10954 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10955 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10956 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10957 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10958#endif
10959 }
10960
10961 /*
10962 * TR.
10963 */
10964 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10965 /* 64-bit capable CPUs. */
10966#if HC_ARCH_BITS == 64
10967 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10968#endif
10969 if (fLongModeGuest)
10970 {
10971 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10972 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10973 }
10974 else
10975 {
10976 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10977 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10978 VMX_IGS_TR_ATTR_TYPE_INVALID);
10979 }
10980 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10981 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10982 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10983 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10984 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10985 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10986 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10987 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10988
10989 /*
10990 * GDTR and IDTR.
10991 */
10992#if HC_ARCH_BITS == 64
10993 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10994 AssertRCBreak(rc);
10995 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10996
10997 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10998 AssertRCBreak(rc);
10999 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
11000#endif
11001
11002 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
11003 AssertRCBreak(rc);
11004 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11005
11006 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
11007 AssertRCBreak(rc);
11008 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11009
11010 /*
11011 * Guest Non-Register State.
11012 */
11013 /* Activity State. */
11014 uint32_t u32ActivityState;
11015 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
11016 AssertRCBreak(rc);
11017 HMVMX_CHECK_BREAK( !u32ActivityState
11018 || (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
11019 VMX_IGS_ACTIVITY_STATE_INVALID);
11020 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
11021 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
11022 uint32_t u32IntrState;
11023 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
11024 AssertRCBreak(rc);
11025 if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
11026 || u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
11027 {
11028 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11029 }
11030
11031 /** @todo Activity state and injecting interrupts. Left as a todo since we
11032 * currently don't use activity states but ACTIVE. */
11033
11034 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
11035 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11036
11037 /* Guest interruptibility-state. */
11038 HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11039 HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
11040 != (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
11041 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11042 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11043 || !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
11044 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11045 if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
11046 {
11047 if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
11048 {
11049 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
11050 && !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
11051 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11052 }
11053 else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
11054 {
11055 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
11056 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11057 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
11058 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11059 }
11060 }
11061 /** @todo Assumes the processor is not in SMM. */
11062 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
11063 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11064 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
11065 || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
11066 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11067 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
11068 && VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
11069 && VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
11070 {
11071 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
11072 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11073 }
11074
11075 /* Pending debug exceptions. */
11076#if HC_ARCH_BITS == 64
11077 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
11078 AssertRCBreak(rc);
11079 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11080 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11081 u32Val = u64Val; /* For pending debug exceptions checks below. */
11082#else
11083 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
11084 AssertRCBreak(rc);
11085 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11086 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11087#endif
11088
11089 if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
11090 || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
11091 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11092 {
11093 if ( (u32Eflags & X86_EFL_TF)
11094 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11095 {
11096 /* Bit 14 is PendingDebug.BS. */
11097 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11098 }
11099 if ( !(u32Eflags & X86_EFL_TF)
11100 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11101 {
11102 /* Bit 14 is PendingDebug.BS. */
11103 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11104 }
11105 }
11106
11107 /* VMCS link pointer. */
11108 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11109 AssertRCBreak(rc);
11110 if (u64Val != UINT64_C(0xffffffffffffffff))
11111 {
11112 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11113 /** @todo Bits beyond the processor's physical-address width MBZ. */
11114 /** @todo 32-bit located in memory referenced by value of this field (as a
11115 * physical address) must contain the processor's VMCS revision ID. */
11116 /** @todo SMM checks. */
11117 }
11118
11119 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11120 * not using Nested Paging? */
11121 if ( pVM->hm.s.fNestedPaging
11122 && !fLongModeGuest
11123 && CPUMIsGuestInPAEModeEx(pCtx))
11124 {
11125 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11126 AssertRCBreak(rc);
11127 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11128
11129 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11130 AssertRCBreak(rc);
11131 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11132
11133 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11134 AssertRCBreak(rc);
11135 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11136
11137 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11138 AssertRCBreak(rc);
11139 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11140 }
11141
11142 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11143 if (uError == VMX_IGS_ERROR)
11144 uError = VMX_IGS_REASON_NOT_FOUND;
11145 } while (0);
11146
11147 pVCpu->hm.s.u32HMError = uError;
11148 return uError;
11149
11150#undef HMVMX_ERROR_BREAK
11151#undef HMVMX_CHECK_BREAK
11152}
11153
11154
11155/** @name VM-exit handlers.
11156 * @{
11157 */
11158/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11159/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11160/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11161
11162/**
11163 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11164 */
11165HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11166{
11167 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11168 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11169 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11170 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11171 return VINF_SUCCESS;
11172 return VINF_EM_RAW_INTERRUPT;
11173}
11174
11175
11176/**
11177 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11178 */
11179HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11180{
11181 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11182 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11183
11184 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11185 AssertRCReturn(rc, rc);
11186
11187 uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
11188 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
11189 && uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
11190 Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11191
11192 if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
11193 {
11194 /*
11195 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
11196 * injected it ourselves and anything we inject is not going to cause a VM-exit directly
11197 * for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
11198 *
11199 * [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11200 * [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
11201 */
11202 VMXDispatchHostNmi();
11203 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11204 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11205 return VINF_SUCCESS;
11206 }
11207
11208 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11209 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
11210 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11211 { /* likely */ }
11212 else
11213 {
11214 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11215 rcStrictRc1 = VINF_SUCCESS;
11216 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11217 return rcStrictRc1;
11218 }
11219
11220 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11221 uint32_t uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
11222 switch (uIntType)
11223 {
11224 case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11225 Assert(uVector == X86_XCPT_DB);
11226 RT_FALL_THRU();
11227 case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11228 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
11229 RT_FALL_THRU();
11230 case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11231 {
11232 /*
11233 * If there's any exception caused as a result of event injection, the resulting
11234 * secondary/final execption will be pending, we shall continue guest execution
11235 * after injecting the event. The page-fault case is complicated and we manually
11236 * handle any currently pending event in hmR0VmxExitXcptPF.
11237 */
11238 if (!pVCpu->hm.s.Event.fPending)
11239 { /* likely */ }
11240 else if (uVector != X86_XCPT_PF)
11241 {
11242 rc = VINF_SUCCESS;
11243 break;
11244 }
11245
11246 switch (uVector)
11247 {
11248 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
11249 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
11250 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
11251 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
11252 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
11253 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
11254
11255 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11256 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11257 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11258 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11259 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11260 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11261 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11262 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11263 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11264 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11265 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11266 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11267 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11268 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11269 default:
11270 {
11271 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11272 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11273 {
11274 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11275 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11276 Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
11277
11278 rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0);
11279 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11280 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11281 AssertRCReturn(rc, rc);
11282 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11283 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11284 0 /* GCPtrFaultAddress */);
11285 }
11286 else
11287 {
11288 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11289 pVCpu->hm.s.u32HMError = uVector;
11290 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11291 }
11292 break;
11293 }
11294 }
11295 break;
11296 }
11297
11298 default:
11299 {
11300 pVCpu->hm.s.u32HMError = uExitIntInfo;
11301 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11302 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
11303 break;
11304 }
11305 }
11306 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11307 return rc;
11308}
11309
11310
11311/**
11312 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11313 */
11314HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11315{
11316 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11317
11318 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11319 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11320
11321 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11322 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11323 return VINF_SUCCESS;
11324}
11325
11326
11327/**
11328 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11329 */
11330HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11331{
11332 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11333 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)))
11334 {
11335 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11336 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11337 }
11338
11339 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11340
11341 /*
11342 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11343 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11344 */
11345 uint32_t fIntrState = 0;
11346 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
11347 AssertRCReturn(rc, rc);
11348
11349 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
11350 if ( fBlockSti
11351 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11352 {
11353 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11354 }
11355
11356 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11357 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11358
11359 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11360 return VINF_SUCCESS;
11361}
11362
11363
11364/**
11365 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11366 */
11367HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11368{
11369 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11370 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11371}
11372
11373
11374/**
11375 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11376 */
11377HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11378{
11379 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11380 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11381}
11382
11383
11384/**
11385 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11386 */
11387HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11388{
11389 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11390
11391 /*
11392 * Get the state we need and update the exit history entry.
11393 */
11394 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11395 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
11396 AssertRCReturn(rc, rc);
11397
11398 VBOXSTRICTRC rcStrict;
11399 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11400 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
11401 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11402 if (!pExitRec)
11403 {
11404 /*
11405 * Regular CPUID instruction execution.
11406 */
11407 rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
11408 if (rcStrict == VINF_SUCCESS)
11409 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11410 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11411 {
11412 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11413 rcStrict = VINF_SUCCESS;
11414 }
11415 }
11416 else
11417 {
11418 /*
11419 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
11420 */
11421 int rc2 = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11422 AssertRCReturn(rc2, rc2);
11423
11424 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11425 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11426
11427 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11428 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11429
11430 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11431 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11432 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11433 }
11434 return rcStrict;
11435}
11436
11437
11438/**
11439 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11440 */
11441HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11442{
11443 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11444 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR4);
11445 AssertRCReturn(rc, rc);
11446
11447 if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
11448 return VINF_EM_RAW_EMULATE_INSTR;
11449
11450 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11451 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11452}
11453
11454
11455/**
11456 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11457 */
11458HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11459{
11460 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11461 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11462 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11463 AssertRCReturn(rc, rc);
11464
11465 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11466 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11467 {
11468 /* If we get a spurious VM-exit when offsetting is enabled,
11469 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11470 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
11471 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11472 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11473 }
11474 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11475 {
11476 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11477 rcStrict = VINF_SUCCESS;
11478 }
11479 return rcStrict;
11480}
11481
11482
11483/**
11484 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11485 */
11486HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11487{
11488 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11489 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_TSC_AUX);
11490 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11491 AssertRCReturn(rc, rc);
11492
11493 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11494 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11495 {
11496 /* If we get a spurious VM-exit when offsetting is enabled,
11497 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11498 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
11499 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11500 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11501 }
11502 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11503 {
11504 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11505 rcStrict = VINF_SUCCESS;
11506 }
11507 return rcStrict;
11508}
11509
11510
11511/**
11512 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11513 */
11514HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11515{
11516 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11517 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11518 AssertRCReturn(rc, rc);
11519
11520 PVM pVM = pVCpu->CTX_SUFF(pVM);
11521 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11522 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11523 if (RT_LIKELY(rc == VINF_SUCCESS))
11524 {
11525 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11526 Assert(pVmxTransient->cbInstr == 2);
11527 }
11528 else
11529 {
11530 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11531 rc = VERR_EM_INTERPRETER;
11532 }
11533 return rc;
11534}
11535
11536
11537/**
11538 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11539 */
11540HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11541{
11542 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11543
11544 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11545 if (EMAreHypercallInstructionsEnabled(pVCpu))
11546 {
11547 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CR0
11548 | CPUMCTX_EXTRN_SS | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_EFER);
11549 AssertRCReturn(rc, rc);
11550
11551 /* Perform the hypercall. */
11552 rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
11553 if (rcStrict == VINF_SUCCESS)
11554 {
11555 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11556 AssertRCReturn(rc, rc);
11557 }
11558 else
11559 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11560 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11561 || RT_FAILURE(rcStrict));
11562
11563 /* If the hypercall changes anything other than guest's general-purpose registers,
11564 we would need to reload the guest changed bits here before VM-entry. */
11565 }
11566 else
11567 Log4Func(("Hypercalls not enabled\n"));
11568
11569 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11570 if (RT_FAILURE(rcStrict))
11571 {
11572 hmR0VmxSetPendingXcptUD(pVCpu);
11573 rcStrict = VINF_SUCCESS;
11574 }
11575
11576 return rcStrict;
11577}
11578
11579
11580/**
11581 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11582 */
11583HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11584{
11585 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11586 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11587
11588 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11589 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11590 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
11591 AssertRCReturn(rc, rc);
11592
11593 VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
11594
11595 if (rcStrict == VINF_SUCCESS || rcStrict == VINF_PGM_SYNC_CR3)
11596 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11597 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11598 {
11599 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11600 rcStrict = VINF_SUCCESS;
11601 }
11602 else
11603 AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) sttus: %Rrc\n", pVmxTransient->uExitQual,
11604 VBOXSTRICTRC_VAL(rcStrict)));
11605 return rcStrict;
11606}
11607
11608
11609/**
11610 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11611 */
11612HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11613{
11614 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11615 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11616 AssertRCReturn(rc, rc);
11617
11618 PVM pVM = pVCpu->CTX_SUFF(pVM);
11619 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11620 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11621 if (RT_LIKELY(rc == VINF_SUCCESS))
11622 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11623 else
11624 {
11625 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11626 rc = VERR_EM_INTERPRETER;
11627 }
11628 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11629 return rc;
11630}
11631
11632
11633/**
11634 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11635 */
11636HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11637{
11638 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11639 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11640 AssertRCReturn(rc, rc);
11641
11642 PVM pVM = pVCpu->CTX_SUFF(pVM);
11643 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11644 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11645 rc = VBOXSTRICTRC_VAL(rc2);
11646 if (RT_LIKELY( rc == VINF_SUCCESS
11647 || rc == VINF_EM_HALT))
11648 {
11649 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11650 AssertRCReturn(rc3, rc3);
11651
11652 if ( rc == VINF_EM_HALT
11653 && EMMonitorWaitShouldContinue(pVCpu, pCtx))
11654 rc = VINF_SUCCESS;
11655 }
11656 else
11657 {
11658 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11659 rc = VERR_EM_INTERPRETER;
11660 }
11661 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11662 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11663 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11664 return rc;
11665}
11666
11667
11668/**
11669 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11670 */
11671HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11672{
11673 /*
11674 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11675 * mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11676 * treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11677 * VMX root operation. If we get here, something funny is going on.
11678 *
11679 * See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11680 */
11681 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11682 AssertMsgFailed(("Unexpected RSM VM-exit\n"));
11683 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11684}
11685
11686
11687/**
11688 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11689 */
11690HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11691{
11692 /*
11693 * This can only happen if we support dual-monitor treatment of SMI, which can be activated
11694 * by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11695 * this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11696 * an SMI. If we get here, something funny is going on.
11697 *
11698 * See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11699 * See Intel spec. 25.3 "Other Causes of VM-Exits"
11700 */
11701 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11702 AssertMsgFailed(("Unexpected SMI VM-exit\n"));
11703 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11704}
11705
11706
11707/**
11708 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11709 */
11710HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11711{
11712 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11713 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11714 AssertMsgFailed(("Unexpected IO SMI VM-exit\n"));
11715 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11716}
11717
11718
11719/**
11720 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11721 */
11722HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11723{
11724 /*
11725 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11726 * We don't make use of it as our guests don't have direct access to the host LAPIC.
11727 * See Intel spec. 25.3 "Other Causes of VM-exits".
11728 */
11729 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11730 AssertMsgFailed(("Unexpected SIPI VM-exit\n"));
11731 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11732}
11733
11734
11735/**
11736 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11737 * VM-exit.
11738 */
11739HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11740{
11741 /*
11742 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11743 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11744 *
11745 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11746 * See Intel spec. "23.8 Restrictions on VMX operation".
11747 */
11748 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11749 return VINF_SUCCESS;
11750}
11751
11752
11753/**
11754 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11755 * VM-exit.
11756 */
11757HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11758{
11759 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11760 return VINF_EM_RESET;
11761}
11762
11763
11764/**
11765 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11766 */
11767HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11768{
11769 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11770 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_HLT_EXIT);
11771
11772 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11773 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RFLAGS);
11774 AssertRCReturn(rc, rc);
11775
11776 if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
11777 rc = VINF_SUCCESS;
11778 else
11779 rc = VINF_EM_HALT;
11780
11781 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11782 if (rc != VINF_SUCCESS)
11783 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11784 return rc;
11785}
11786
11787
11788/**
11789 * VM-exit handler for instructions that result in a \#UD exception delivered to
11790 * the guest.
11791 */
11792HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11793{
11794 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11795 hmR0VmxSetPendingXcptUD(pVCpu);
11796 return VINF_SUCCESS;
11797}
11798
11799
11800/**
11801 * VM-exit handler for expiry of the VMX preemption timer.
11802 */
11803HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11804{
11805 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11806
11807 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11808 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11809
11810 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11811 PVM pVM = pVCpu->CTX_SUFF(pVM);
11812 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11813 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11814 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11815}
11816
11817
11818/**
11819 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11820 */
11821HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11822{
11823 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11824
11825 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11826 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_CR4);
11827 AssertRCReturn(rc, rc);
11828
11829 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11830 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11831 : HM_CHANGED_RAISED_XCPT_MASK);
11832
11833 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11834 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
11835
11836 return rcStrict;
11837}
11838
11839
11840/**
11841 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11842 */
11843HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11844{
11845 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11846 /** @todo Use VM-exit instruction information. */
11847 return VERR_EM_INTERPRETER;
11848}
11849
11850
11851/**
11852 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11853 * Error VM-exit.
11854 */
11855HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11856{
11857 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11858 AssertRCReturn(rc, rc);
11859 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11860 if (RT_FAILURE(rc))
11861 return rc;
11862
11863 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
11864 NOREF(uInvalidReason);
11865
11866#ifdef VBOX_STRICT
11867 uint32_t fIntrState;
11868 RTHCUINTREG uHCReg;
11869 uint64_t u64Val;
11870 uint32_t u32Val;
11871
11872 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11873 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11874 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11875 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
11876 AssertRCReturn(rc, rc);
11877
11878 Log4(("uInvalidReason %u\n", uInvalidReason));
11879 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11880 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11881 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11882 Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
11883
11884 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11885 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11886 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11887 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11888 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11889 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11890 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11891 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11892 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11893 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11894 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11895 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11896
11897 hmR0DumpRegs(pVCpu);
11898#else
11899 NOREF(pVmxTransient);
11900#endif
11901
11902 return VERR_VMX_INVALID_GUEST_STATE;
11903}
11904
11905
11906/**
11907 * VM-exit handler for VM-entry failure due to an MSR-load
11908 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11909 */
11910HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11911{
11912 AssertMsgFailed(("Unexpected MSR-load exit\n"));
11913 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11914}
11915
11916
11917/**
11918 * VM-exit handler for VM-entry failure due to a machine-check event
11919 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11920 */
11921HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11922{
11923 AssertMsgFailed(("Unexpected machine-check event exit\n"));
11924 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11925}
11926
11927
11928/**
11929 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11930 * theory.
11931 */
11932HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11933{
11934 RT_NOREF2(pVCpu, pVmxTransient);
11935 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d\n", pVmxTransient->uExitReason));
11936 return VERR_VMX_UNDEFINED_EXIT_CODE;
11937}
11938
11939
11940/**
11941 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11942 * (VMX_EXIT_GDTR_IDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11943 * Conditional VM-exit.
11944 */
11945HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11946{
11947 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11948
11949 /* By default, we don't enable VMX_PROC_CTLS2_DESCRIPTOR_TABLE_EXIT. */
11950 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11951 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_DESC_TABLE_EXIT)
11952 return VERR_EM_INTERPRETER;
11953 AssertMsgFailed(("Unexpected XDTR access\n"));
11954 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11955}
11956
11957
11958/**
11959 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11960 */
11961HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11962{
11963 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11964
11965 /* By default, we don't enable VMX_PROC_CTLS2_RDRAND_EXIT. */
11966 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_RDRAND_EXIT)
11967 return VERR_EM_INTERPRETER;
11968 AssertMsgFailed(("Unexpected RDRAND exit\n"));
11969 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11970}
11971
11972
11973/**
11974 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11975 */
11976HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11977{
11978 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11979
11980 /** @todo Optimize this: We currently drag in in the whole MSR state
11981 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11982 * MSRs required. That would require changes to IEM and possibly CPUM too.
11983 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11984 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; NOREF(idMsr); /* Save it. */
11985 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11986 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
11987 AssertRCReturn(rc, rc);
11988
11989 Log4Func(("ecx=%#RX32\n", idMsr));
11990
11991#ifdef VBOX_STRICT
11992 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
11993 {
11994 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr)
11995 && idMsr != MSR_K6_EFER)
11996 {
11997 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
11998 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11999 }
12000 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
12001 {
12002 VMXMSREXITREAD enmRead;
12003 VMXMSREXITWRITE enmWrite;
12004 int rc2 = hmR0VmxGetMsrPermission(pVCpu, idMsr, &enmRead, &enmWrite);
12005 AssertRCReturn(rc2, rc2);
12006 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
12007 {
12008 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
12009 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12010 }
12011 }
12012 }
12013#endif
12014
12015 VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
12016 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
12017 if (rcStrict == VINF_SUCCESS)
12018 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
12019 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX);
12020 else if (rcStrict == VINF_IEM_RAISED_XCPT)
12021 {
12022 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
12023 rcStrict = VINF_SUCCESS;
12024 }
12025 else
12026 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12027
12028 return rcStrict;
12029}
12030
12031
12032/**
12033 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12034 */
12035HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12036{
12037 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12038
12039 /** @todo Optimize this: We currently drag in in the whole MSR state
12040 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
12041 * MSRs required. That would require changes to IEM and possibly CPUM too.
12042 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
12043 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; /* Save it. */
12044 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12045 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
12046 AssertRCReturn(rc, rc);
12047
12048 Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
12049
12050 VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
12051 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12052
12053 if (rcStrict == VINF_SUCCESS)
12054 {
12055 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12056
12057 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
12058 if ( idMsr == MSR_IA32_APICBASE
12059 || ( idMsr >= MSR_IA32_X2APIC_START
12060 && idMsr <= MSR_IA32_X2APIC_END))
12061 {
12062 /*
12063 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12064 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before IEM changes it.
12065 */
12066 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
12067 }
12068 else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12069 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12070 else if (idMsr == MSR_K6_EFER)
12071 {
12072 /*
12073 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12074 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12075 * the other bits as well, SCE and NXE. See @bugref{7368}.
12076 */
12077 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS
12078 | HM_CHANGED_VMX_EXIT_CTLS);
12079 }
12080
12081 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12082 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
12083 {
12084 switch (idMsr)
12085 {
12086 case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
12087 case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
12088 case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
12089 case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
12090 case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
12091 case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
12092 default:
12093 {
12094 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
12095 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12096 else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
12097 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
12098 break;
12099 }
12100 }
12101 }
12102#ifdef VBOX_STRICT
12103 else
12104 {
12105 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12106 switch (idMsr)
12107 {
12108 case MSR_IA32_SYSENTER_CS:
12109 case MSR_IA32_SYSENTER_EIP:
12110 case MSR_IA32_SYSENTER_ESP:
12111 case MSR_K8_FS_BASE:
12112 case MSR_K8_GS_BASE:
12113 {
12114 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
12115 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12116 }
12117
12118 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12119 default:
12120 {
12121 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
12122 {
12123 /* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
12124 if (idMsr != MSR_K6_EFER)
12125 {
12126 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12127 idMsr));
12128 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12129 }
12130 }
12131
12132 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
12133 {
12134 VMXMSREXITREAD enmRead;
12135 VMXMSREXITWRITE enmWrite;
12136 int rc2 = hmR0VmxGetMsrPermission(pVCpu, idMsr, &enmRead, &enmWrite);
12137 AssertRCReturn(rc2, rc2);
12138 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12139 {
12140 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
12141 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12142 }
12143 }
12144 break;
12145 }
12146 }
12147 }
12148#endif /* VBOX_STRICT */
12149 }
12150 else if (rcStrict == VINF_IEM_RAISED_XCPT)
12151 {
12152 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
12153 rcStrict = VINF_SUCCESS;
12154 }
12155 else
12156 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12157
12158 return rcStrict;
12159}
12160
12161
12162/**
12163 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12164 */
12165HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12166{
12167 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12168 /** @todo The guest has likely hit a contended spinlock. We might want to
12169 * poke a schedule different guest VCPU. */
12170 return VINF_EM_RAW_INTERRUPT;
12171}
12172
12173
12174/**
12175 * VM-exit handler for when the TPR value is lowered below the specified
12176 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12177 */
12178HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12179{
12180 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12181 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
12182
12183 /*
12184 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12185 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12186 */
12187 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12188 return VINF_SUCCESS;
12189}
12190
12191
12192/**
12193 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12194 * VM-exit.
12195 *
12196 * @retval VINF_SUCCESS when guest execution can continue.
12197 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12198 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12199 * interpreter.
12200 */
12201HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12202{
12203 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12204 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12205
12206 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12207 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12208 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12209 AssertRCReturn(rc, rc);
12210
12211 VBOXSTRICTRC rcStrict;
12212 PVM pVM = pVCpu->CTX_SUFF(pVM);
12213 RTGCUINTPTR const uExitQual = pVmxTransient->uExitQual;
12214 uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
12215 switch (uAccessType)
12216 {
12217 case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
12218 {
12219 uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
12220 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
12221 VMX_EXIT_QUAL_CRX_GENREG(uExitQual));
12222 AssertMsg( rcStrict == VINF_SUCCESS
12223 || rcStrict == VINF_IEM_RAISED_XCPT
12224 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12225
12226 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
12227 {
12228 case 0:
12229 {
12230 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12231 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12232 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
12233 Log4Func(("CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
12234
12235 /*
12236 * This is a kludge for handling switches back to real mode when we try to use
12237 * V86 mode to run real mode code directly. Problem is that V86 mode cannot
12238 * deal with special selector values, so we have to return to ring-3 and run
12239 * there till the selector values are V86 mode compatible.
12240 *
12241 * Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
12242 * latter is an alias for VINF_IEM_RAISED_XCPT which is converted to VINF_SUCCESs
12243 * at the end of this function.
12244 */
12245 if ( rc == VINF_SUCCESS
12246 && !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
12247 && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
12248 && (uOldCr0 & X86_CR0_PE)
12249 && !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
12250 {
12251 /** @todo check selectors rather than returning all the time. */
12252 Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
12253 rcStrict = VINF_EM_RESCHEDULE_REM;
12254 }
12255 break;
12256 }
12257
12258 case 2:
12259 {
12260 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
12261 /* Nothing to do here, CR2 it's not part of the VMCS. */
12262 break;
12263 }
12264
12265 case 3:
12266 {
12267 Assert( !pVM->hm.s.fNestedPaging
12268 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
12269 || pVCpu->hm.s.fUsingDebugLoop);
12270 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
12271 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12272 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR3);
12273 Log4Func(("CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
12274 break;
12275 }
12276
12277 case 4:
12278 {
12279 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
12280 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12281 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR4);
12282 Log4Func(("CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
12283 pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12284 break;
12285 }
12286
12287 case 8:
12288 {
12289 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
12290 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
12291 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12292 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_APIC_TPR);
12293 break;
12294 }
12295 default:
12296 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual)));
12297 break;
12298 }
12299 break;
12300 }
12301
12302 case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
12303 {
12304 Assert( !pVM->hm.s.fNestedPaging
12305 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
12306 || pVCpu->hm.s.fUsingDebugLoop
12307 || VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 3);
12308 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12309 Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 8
12310 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
12311
12312 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_GENREG(uExitQual),
12313 VMX_EXIT_QUAL_CRX_REGISTER(uExitQual));
12314 AssertMsg( rcStrict == VINF_SUCCESS
12315 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12316#ifdef VBOX_WITH_STATISTICS
12317 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
12318 {
12319 case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
12320 case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
12321 case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
12322 case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
12323 case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
12324 }
12325#endif
12326 Log4Func(("CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
12327 VBOXSTRICTRC_VAL(rcStrict)));
12328 if (VMX_EXIT_QUAL_CRX_GENREG(uExitQual) == X86_GREG_xSP)
12329 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RSP);
12330 else
12331 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12332 break;
12333 }
12334
12335 case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12336 {
12337 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12338 AssertMsg( rcStrict == VINF_SUCCESS
12339 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12340
12341 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12342 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12343 Log4Func(("CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12344 break;
12345 }
12346
12347 case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12348 {
12349 /* Note! LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here. */
12350 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual));
12351 AssertMsg( rcStrict == VINF_SUCCESS
12352 || rcStrict == VINF_IEM_RAISED_XCPT
12353 , ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12354
12355 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12356 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12357 Log4Func(("LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12358 break;
12359 }
12360
12361 default:
12362 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12363 VERR_VMX_UNEXPECTED_EXCEPTION);
12364 }
12365
12366 Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS))
12367 == (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS));
12368 if (rcStrict == VINF_IEM_RAISED_XCPT)
12369 {
12370 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
12371 rcStrict = VINF_SUCCESS;
12372 }
12373
12374 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12375 NOREF(pVM);
12376 return rcStrict;
12377}
12378
12379
12380/**
12381 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12382 * VM-exit.
12383 */
12384HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12385{
12386 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12387 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12388
12389 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12390 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12391 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12392 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_EFER);
12393 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12394 AssertRCReturn(rc, rc);
12395
12396 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12397 uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
12398 uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQual);
12399 bool fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12400 bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
12401 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
12402 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12403 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12404
12405 /*
12406 * Update exit history to see if this exit can be optimized.
12407 */
12408 VBOXSTRICTRC rcStrict;
12409 PCEMEXITREC pExitRec = NULL;
12410 if ( !fGstStepping
12411 && !fDbgStepping)
12412 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12413 !fIOString
12414 ? !fIOWrite
12415 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12416 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12417 : !fIOWrite
12418 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12419 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12420 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12421 if (!pExitRec)
12422 {
12423 /* I/O operation lookup arrays. */
12424 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12425 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12426 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12427 uint32_t const cbInstr = pVmxTransient->cbInstr;
12428 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12429 PVM pVM = pVCpu->CTX_SUFF(pVM);
12430 if (fIOString)
12431 {
12432 /*
12433 * INS/OUTS - I/O String instruction.
12434 *
12435 * Use instruction-information if available, otherwise fall back on
12436 * interpreting the instruction.
12437 */
12438 Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12439 AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
12440 bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
12441 if (fInsOutsInfo)
12442 {
12443 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12444 AssertRCReturn(rc2, rc2);
12445 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12446 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12447 IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12448 bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
12449 if (fIOWrite)
12450 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12451 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12452 else
12453 {
12454 /*
12455 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12456 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12457 * See Intel Instruction spec. for "INS".
12458 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12459 */
12460 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12461 }
12462 }
12463 else
12464 rcStrict = IEMExecOne(pVCpu);
12465
12466 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12467 fUpdateRipAlready = true;
12468 }
12469 else
12470 {
12471 /*
12472 * IN/OUT - I/O instruction.
12473 */
12474 Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12475 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12476 Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
12477 if (fIOWrite)
12478 {
12479 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
12480 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12481 if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
12482 && !pCtx->eflags.Bits.u1TF)
12483 rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
12484 }
12485 else
12486 {
12487 uint32_t u32Result = 0;
12488 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12489 if (IOM_SUCCESS(rcStrict))
12490 {
12491 /* Save result of I/O IN instr. in AL/AX/EAX. */
12492 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12493 }
12494 if ( rcStrict == VINF_IOM_R3_IOPORT_READ
12495 && !pCtx->eflags.Bits.u1TF)
12496 rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
12497 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12498 }
12499 }
12500
12501 if (IOM_SUCCESS(rcStrict))
12502 {
12503 if (!fUpdateRipAlready)
12504 {
12505 hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
12506 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12507 }
12508
12509 /*
12510 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12511 * while booting Fedora 17 64-bit guest.
12512 *
12513 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12514 */
12515 if (fIOString)
12516 {
12517 /** @todo Single-step for INS/OUTS with REP prefix? */
12518 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12519 }
12520 else if ( !fDbgStepping
12521 && fGstStepping)
12522 {
12523 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12524 AssertRCReturn(rc, rc);
12525 }
12526
12527 /*
12528 * If any I/O breakpoints are armed, we need to check if one triggered
12529 * and take appropriate action.
12530 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12531 */
12532 rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_DR7);
12533 AssertRCReturn(rc, rc);
12534
12535 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12536 * execution engines about whether hyper BPs and such are pending. */
12537 uint32_t const uDr7 = pCtx->dr[7];
12538 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12539 && X86_DR7_ANY_RW_IO(uDr7)
12540 && (pCtx->cr4 & X86_CR4_DE))
12541 || DBGFBpIsHwIoArmed(pVM)))
12542 {
12543 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12544
12545 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12546 VMMRZCallRing3Disable(pVCpu);
12547 HM_DISABLE_PREEMPT(pVCpu);
12548
12549 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12550
12551 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
12552 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12553 {
12554 /* Raise #DB. */
12555 if (fIsGuestDbgActive)
12556 ASMSetDR6(pCtx->dr[6]);
12557 if (pCtx->dr[7] != uDr7)
12558 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR7;
12559
12560 hmR0VmxSetPendingXcptDB(pVCpu);
12561 }
12562 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12563 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12564 else if ( rcStrict2 != VINF_SUCCESS
12565 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12566 rcStrict = rcStrict2;
12567 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12568
12569 HM_RESTORE_PREEMPT();
12570 VMMRZCallRing3Enable(pVCpu);
12571 }
12572 }
12573
12574#ifdef VBOX_STRICT
12575 if ( rcStrict == VINF_IOM_R3_IOPORT_READ
12576 || rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
12577 Assert(!fIOWrite);
12578 else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
12579 || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
12580 || rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
12581 Assert(fIOWrite);
12582 else
12583 {
12584# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12585 * statuses, that the VMM device and some others may return. See
12586 * IOM_SUCCESS() for guidance. */
12587 AssertMsg( RT_FAILURE(rcStrict)
12588 || rcStrict == VINF_SUCCESS
12589 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12590 || rcStrict == VINF_EM_DBG_BREAKPOINT
12591 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12592 || rcStrict == VINF_EM_RAW_TO_R3
12593 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12594# endif
12595 }
12596#endif
12597 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12598 }
12599 else
12600 {
12601 /*
12602 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12603 */
12604 int rc2 = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12605 AssertRCReturn(rc2, rc2);
12606 STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12607 : fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12608 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12609 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12610 VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
12611 fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12612
12613 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12614 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12615
12616 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12617 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12618 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12619 }
12620 return rcStrict;
12621}
12622
12623
12624/**
12625 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12626 * VM-exit.
12627 */
12628HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12629{
12630 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12631
12632 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12633 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12634 AssertRCReturn(rc, rc);
12635 if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12636 {
12637 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12638 AssertRCReturn(rc, rc);
12639 if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
12640 {
12641 uint32_t uErrCode;
12642 RTGCUINTPTR GCPtrFaultAddress;
12643 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12644 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12645 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
12646 if (fErrorCodeValid)
12647 {
12648 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12649 AssertRCReturn(rc, rc);
12650 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12651 }
12652 else
12653 uErrCode = 0;
12654
12655 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12656 && uVector == X86_XCPT_PF)
12657 GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
12658 else
12659 GCPtrFaultAddress = 0;
12660
12661 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12662 AssertRCReturn(rc, rc);
12663
12664 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12665 pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
12666
12667 Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
12668 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12669 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12670 }
12671 }
12672
12673 /* Fall back to the interpreter to emulate the task-switch. */
12674 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12675 return VERR_EM_INTERPRETER;
12676}
12677
12678
12679/**
12680 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12681 */
12682HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12683{
12684 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12685 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_MONITOR_TRAP_FLAG);
12686 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
12687 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12688 AssertRCReturn(rc, rc);
12689 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12690 return VINF_EM_DBG_STEPPED;
12691}
12692
12693
12694/**
12695 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12696 */
12697HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12698{
12699 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12700
12701 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12702
12703 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12704 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12705 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12706 {
12707 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12708 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12709 {
12710 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12711 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12712 }
12713 }
12714 else
12715 {
12716 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12717 rcStrict1 = VINF_SUCCESS;
12718 return rcStrict1;
12719 }
12720
12721 /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12722 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12723 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12724 AssertRCReturn(rc, rc);
12725
12726 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12727 uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
12728 VBOXSTRICTRC rcStrict2;
12729 switch (uAccessType)
12730 {
12731 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12732 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12733 {
12734 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
12735 || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
12736 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12737
12738 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12739 GCPhys &= PAGE_BASE_GC_MASK;
12740 GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
12741 PVM pVM = pVCpu->CTX_SUFF(pVM);
12742 Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12743 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
12744
12745 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12746 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12747 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12748 CPUMCTX2CORE(pCtx), GCPhys);
12749 Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12750 if ( rcStrict2 == VINF_SUCCESS
12751 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12752 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12753 {
12754 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12755 | HM_CHANGED_GUEST_APIC_TPR);
12756 rcStrict2 = VINF_SUCCESS;
12757 }
12758 break;
12759 }
12760
12761 default:
12762 Log4Func(("uAccessType=%#x\n", uAccessType));
12763 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12764 break;
12765 }
12766
12767 if (rcStrict2 != VINF_SUCCESS)
12768 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12769 return rcStrict2;
12770}
12771
12772
12773/**
12774 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12775 * VM-exit.
12776 */
12777HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12778{
12779 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12780
12781 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12782 if (pVmxTransient->fWasGuestDebugStateActive)
12783 {
12784 AssertMsgFailed(("Unexpected MOV DRx exit\n"));
12785 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12786 }
12787
12788 if ( !pVCpu->hm.s.fSingleInstruction
12789 && !pVmxTransient->fWasHyperDebugStateActive)
12790 {
12791 Assert(!DBGFIsStepping(pVCpu));
12792 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12793
12794 /* Don't intercept MOV DRx any more. */
12795 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
12796 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12797 AssertRCReturn(rc, rc);
12798
12799 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12800 VMMRZCallRing3Disable(pVCpu);
12801 HM_DISABLE_PREEMPT(pVCpu);
12802
12803 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12804 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12805 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12806
12807 HM_RESTORE_PREEMPT();
12808 VMMRZCallRing3Enable(pVCpu);
12809
12810#ifdef VBOX_WITH_STATISTICS
12811 rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12812 AssertRCReturn(rc, rc);
12813 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12814 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12815 else
12816 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12817#endif
12818 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12819 return VINF_SUCCESS;
12820 }
12821
12822 /*
12823 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12824 * Update the segment registers and DR7 from the CPU.
12825 */
12826 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12827 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12828 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_DR7);
12829 AssertRCReturn(rc, rc);
12830 Log4Func(("CS:RIP=%04x:%08RX64\n", pCtx->cs.Sel, pCtx->rip));
12831
12832 PVM pVM = pVCpu->CTX_SUFF(pVM);
12833 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12834 {
12835 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12836 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
12837 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
12838 if (RT_SUCCESS(rc))
12839 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12840 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12841 }
12842 else
12843 {
12844 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12845 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
12846 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
12847 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12848 }
12849
12850 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12851 if (RT_SUCCESS(rc))
12852 {
12853 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12854 AssertRCReturn(rc2, rc2);
12855 return VINF_SUCCESS;
12856 }
12857 return rc;
12858}
12859
12860
12861/**
12862 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12863 * Conditional VM-exit.
12864 */
12865HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12866{
12867 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12868 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12869
12870 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12871 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12872 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12873 {
12874 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12875 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12876 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12877 {
12878 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12879 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12880 }
12881 }
12882 else
12883 {
12884 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12885 rcStrict1 = VINF_SUCCESS;
12886 return rcStrict1;
12887 }
12888
12889 /*
12890 * Get sufficent state and update the exit history entry.
12891 */
12892 RTGCPHYS GCPhys;
12893 int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
12894 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12895 AssertRCReturn(rc, rc);
12896
12897 VBOXSTRICTRC rcStrict;
12898 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12899 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
12900 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12901 if (!pExitRec)
12902 {
12903 /*
12904 * If we succeed, resume guest execution.
12905 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12906 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12907 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12908 * weird case. See @bugref{6043}.
12909 */
12910 PVM pVM = pVCpu->CTX_SUFF(pVM);
12911 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12912 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
12913 Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12914 if ( rcStrict == VINF_SUCCESS
12915 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12916 || rcStrict == VERR_PAGE_NOT_PRESENT)
12917 {
12918 /* Successfully handled MMIO operation. */
12919 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12920 | HM_CHANGED_GUEST_APIC_TPR);
12921 rcStrict = VINF_SUCCESS;
12922 }
12923 }
12924 else
12925 {
12926 /*
12927 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12928 */
12929 int rc2 = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12930 AssertRCReturn(rc2, rc2);
12931
12932 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12933 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12934
12935 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12936 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12937
12938 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12939 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12940 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12941 }
12942 return VBOXSTRICTRC_TODO(rcStrict);
12943}
12944
12945
12946/**
12947 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12948 * VM-exit.
12949 */
12950HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12951{
12952 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12953 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12954
12955 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12956 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12957 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12958 {
12959 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12960 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12961 Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12962 }
12963 else
12964 {
12965 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12966 rcStrict1 = VINF_SUCCESS;
12967 return rcStrict1;
12968 }
12969
12970 RTGCPHYS GCPhys;
12971 int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
12972 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12973 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12974 AssertRCReturn(rc, rc);
12975
12976 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12977 AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
12978
12979 RTGCUINT uErrorCode = 0;
12980 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12981 uErrorCode |= X86_TRAP_PF_ID;
12982 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12983 uErrorCode |= X86_TRAP_PF_RW;
12984 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12985 uErrorCode |= X86_TRAP_PF_P;
12986
12987 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12988
12989
12990 /* Handle the pagefault trap for the nested shadow table. */
12991 PVM pVM = pVCpu->CTX_SUFF(pVM);
12992 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12993
12994 Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
12995 pCtx->cs.Sel, pCtx->rip));
12996
12997 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
12998 TRPMResetTrap(pVCpu);
12999
13000 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
13001 if ( rcStrict2 == VINF_SUCCESS
13002 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13003 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
13004 {
13005 /* Successfully synced our nested page tables. */
13006 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
13007 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
13008 return VINF_SUCCESS;
13009 }
13010
13011 Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
13012 return rcStrict2;
13013}
13014
13015/** @} */
13016
13017/** @name VM-exit exception handlers.
13018 * @{
13019 */
13020/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13021/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
13022/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13023
13024/**
13025 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
13026 */
13027static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13028{
13029 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13030 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13031
13032 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0);
13033 AssertRCReturn(rc, rc);
13034
13035 if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
13036 {
13037 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13038 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13039
13040 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13041 * provides VM-exit instruction length. If this causes problem later,
13042 * disassemble the instruction like it's done on AMD-V. */
13043 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13044 AssertRCReturn(rc2, rc2);
13045 return rc;
13046 }
13047
13048 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13049 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13050 return rc;
13051}
13052
13053
13054/**
13055 * VM-exit exception handler for \#BP (Breakpoint exception).
13056 */
13057static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13058{
13059 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13060 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13061
13062 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13063 AssertRCReturn(rc, rc);
13064
13065 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13066 rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13067 if (rc == VINF_EM_RAW_GUEST_TRAP)
13068 {
13069 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13070 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13071 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13072 AssertRCReturn(rc, rc);
13073
13074 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13075 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13076 }
13077
13078 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
13079 return rc;
13080}
13081
13082
13083/**
13084 * VM-exit exception handler for \#AC (alignment check exception).
13085 */
13086static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13087{
13088 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13089
13090 /*
13091 * Re-inject it. We'll detect any nesting before getting here.
13092 */
13093 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13094 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13095 AssertRCReturn(rc, rc);
13096 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13097
13098 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13099 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13100 return VINF_SUCCESS;
13101}
13102
13103
13104/**
13105 * VM-exit exception handler for \#DB (Debug exception).
13106 */
13107static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13108{
13109 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13110 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13111
13112 /*
13113 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13114 * for processing.
13115 */
13116 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13117
13118 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13119 uint64_t uDR6 = X86_DR6_INIT_VAL;
13120 uDR6 |= (pVmxTransient->uExitQual & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
13121
13122 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13123 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13124 Log6Func(("rc=%Rrc\n", rc));
13125 if (rc == VINF_EM_RAW_GUEST_TRAP)
13126 {
13127 /*
13128 * The exception was for the guest. Update DR6, DR7.GD and
13129 * IA32_DEBUGCTL.LBR before forwarding it.
13130 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13131 */
13132 VMMRZCallRing3Disable(pVCpu);
13133 HM_DISABLE_PREEMPT(pVCpu);
13134
13135 pCtx->dr[6] &= ~X86_DR6_B_MASK;
13136 pCtx->dr[6] |= uDR6;
13137 if (CPUMIsGuestDebugStateActive(pVCpu))
13138 ASMSetDR6(pCtx->dr[6]);
13139
13140 HM_RESTORE_PREEMPT();
13141 VMMRZCallRing3Enable(pVCpu);
13142
13143 rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_DR7);
13144 AssertRCReturn(rc, rc);
13145
13146 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13147 pCtx->dr[7] &= ~X86_DR7_GD;
13148
13149 /* Paranoia. */
13150 pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13151 pCtx->dr[7] |= X86_DR7_RA1_MASK;
13152
13153 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
13154 AssertRCReturn(rc, rc);
13155
13156 /*
13157 * Raise #DB in the guest.
13158 *
13159 * It is important to reflect exactly what the VM-exit gave us (preserving the
13160 * interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
13161 * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
13162 * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
13163 *
13164 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
13165 * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
13166 */
13167 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13168 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13169 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13170 AssertRCReturn(rc, rc);
13171 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13172 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13173 return VINF_SUCCESS;
13174 }
13175
13176 /*
13177 * Not a guest trap, must be a hypervisor related debug event then.
13178 * Update DR6 in case someone is interested in it.
13179 */
13180 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13181 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13182 CPUMSetHyperDR6(pVCpu, uDR6);
13183
13184 return rc;
13185}
13186
13187
13188/**
13189 * VM-exit exception handler for \#GP (General-protection exception).
13190 *
13191 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13192 */
13193static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13194{
13195 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13196 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13197
13198 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13199 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13200 { /* likely */ }
13201 else
13202 {
13203#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13204 Assert(pVCpu->hm.s.fUsingDebugLoop);
13205#endif
13206 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13207 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13208 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13209 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13210 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13211 AssertRCReturn(rc, rc);
13212 Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pCtx->cs.Sel, pCtx->rip,
13213 pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
13214 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13215 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13216 return rc;
13217 }
13218
13219 Assert(CPUMIsGuestInRealModeEx(pCtx));
13220 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13221
13222 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13223 AssertRCReturn(rc, rc);
13224
13225 VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
13226 if (rcStrict == VINF_SUCCESS)
13227 {
13228 if (!CPUMIsGuestInRealModeEx(pCtx))
13229 {
13230 /*
13231 * The guest is no longer in real-mode, check if we can continue executing the
13232 * guest using hardware-assisted VMX. Otherwise, fall back to emulation.
13233 */
13234 if (HMVmxCanExecuteGuest(pVCpu, pCtx))
13235 {
13236 Log4Func(("Mode changed but guest still suitable for executing using VT-x\n"));
13237 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
13238 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13239 }
13240 else
13241 {
13242 Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
13243 rcStrict = VINF_EM_RESCHEDULE;
13244 }
13245 }
13246 else
13247 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13248 }
13249 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13250 {
13251 rcStrict = VINF_SUCCESS;
13252 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13253 }
13254 return VBOXSTRICTRC_VAL(rcStrict);
13255}
13256
13257
13258/**
13259 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13260 * the exception reported in the VMX transient structure back into the VM.
13261 *
13262 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13263 * up-to-date.
13264 */
13265static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13266{
13267 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13268#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13269 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13270 ("uVector=%#x u32XcptBitmap=%#X32\n",
13271 VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13272#endif
13273
13274 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13275 hmR0VmxCheckExitDueToEventDelivery(). */
13276 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13277 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13278 AssertRCReturn(rc, rc);
13279 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13280
13281#ifdef DEBUG_ramshankar
13282 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
13283 uint8_t uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13284 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13285#endif
13286
13287 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13288 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13289 return VINF_SUCCESS;
13290}
13291
13292
13293/**
13294 * VM-exit exception handler for \#PF (Page-fault exception).
13295 */
13296static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13297{
13298 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13299 PVM pVM = pVCpu->CTX_SUFF(pVM);
13300 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13301 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13302 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13303 AssertRCReturn(rc, rc);
13304
13305 if (!pVM->hm.s.fNestedPaging)
13306 { /* likely */ }
13307 else
13308 {
13309#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13310 Assert(pVCpu->hm.s.fUsingDebugLoop);
13311#endif
13312 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13313 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13314 {
13315 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13316 pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
13317 }
13318 else
13319 {
13320 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13321 hmR0VmxSetPendingXcptDF(pVCpu);
13322 Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13323 }
13324 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13325 return rc;
13326 }
13327
13328 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13329 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13330 if (pVmxTransient->fVectoringPF)
13331 {
13332 Assert(pVCpu->hm.s.Event.fPending);
13333 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13334 }
13335
13336 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13337 rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13338 AssertRCReturn(rc, rc);
13339
13340 Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
13341 pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
13342
13343 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13344 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
13345
13346 Log4Func(("#PF: rc=%Rrc\n", rc));
13347 if (rc == VINF_SUCCESS)
13348 {
13349 /*
13350 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13351 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13352 */
13353 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13354 TRPMResetTrap(pVCpu);
13355 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13356 return rc;
13357 }
13358
13359 if (rc == VINF_EM_RAW_GUEST_TRAP)
13360 {
13361 if (!pVmxTransient->fVectoringDoublePF)
13362 {
13363 /* It's a guest page fault and needs to be reflected to the guest. */
13364 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13365 TRPMResetTrap(pVCpu);
13366 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13367 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13368 uGstErrorCode, pVmxTransient->uExitQual);
13369 }
13370 else
13371 {
13372 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13373 TRPMResetTrap(pVCpu);
13374 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13375 hmR0VmxSetPendingXcptDF(pVCpu);
13376 Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13377 }
13378
13379 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13380 return VINF_SUCCESS;
13381 }
13382
13383 TRPMResetTrap(pVCpu);
13384 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13385 return rc;
13386}
13387
13388/** @} */
13389
13390#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13391/** @name Nested-guest VM-exit handlers.
13392 * @{
13393 */
13394/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13395/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= Nested-guest VM-exit handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13396/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13397
13398/**
13399 * VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
13400 */
13401HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13402{
13403 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13404
13405 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13406 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13407 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13408 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13409 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13410 AssertRCReturn(rc, rc);
13411
13412 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13413
13414 VMXVEXITINFO ExitInfo;
13415 RT_ZERO(ExitInfo);
13416 ExitInfo.uReason = pVmxTransient->uExitReason;
13417 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13418 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13419 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13420 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13421
13422 VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
13423 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13424 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13425 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13426 {
13427 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13428 rcStrict = VINF_SUCCESS;
13429 }
13430 return rcStrict;
13431}
13432
13433
13434/**
13435 * VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
13436 */
13437HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13438{
13439 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13440
13441 /** @todo NSTVMX: Vmlaunch. */
13442 hmR0VmxSetPendingXcptUD(pVCpu);
13443 return VINF_SUCCESS;
13444}
13445
13446
13447/**
13448 * VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
13449 */
13450HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13451{
13452 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13453
13454 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13455 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13456 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13457 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13458 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13459 AssertRCReturn(rc, rc);
13460
13461 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13462
13463 VMXVEXITINFO ExitInfo;
13464 RT_ZERO(ExitInfo);
13465 ExitInfo.uReason = pVmxTransient->uExitReason;
13466 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13467 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13468 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13469 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13470
13471 VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
13472 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13473 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13474 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13475 {
13476 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13477 rcStrict = VINF_SUCCESS;
13478 }
13479 return rcStrict;
13480}
13481
13482
13483/**
13484 * VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
13485 */
13486HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13487{
13488 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13489
13490 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13491 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13492 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13493 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13494 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13495 AssertRCReturn(rc, rc);
13496
13497 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13498
13499 VMXVEXITINFO ExitInfo;
13500 RT_ZERO(ExitInfo);
13501 ExitInfo.uReason = pVmxTransient->uExitReason;
13502 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13503 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13504 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13505 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
13506
13507 VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
13508 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13509 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13510 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13511 {
13512 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13513 rcStrict = VINF_SUCCESS;
13514 }
13515 return rcStrict;
13516}
13517
13518
13519/**
13520 * VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Unconditional VM-exit.
13521 */
13522HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13523{
13524 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13525
13526 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13527 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13528 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13529 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13530 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13531 AssertRCReturn(rc, rc);
13532
13533 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13534
13535 VMXVEXITINFO ExitInfo;
13536 RT_ZERO(ExitInfo);
13537 ExitInfo.uReason = pVmxTransient->uExitReason;
13538 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13539 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13540 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13541 if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
13542 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
13543
13544 VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
13545 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13546 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13547 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13548 {
13549 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13550 rcStrict = VINF_SUCCESS;
13551 }
13552 return rcStrict;
13553}
13554
13555
13556/**
13557 * VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
13558 */
13559HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13560{
13561 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13562
13563 /** @todo NSTVMX: Vmresume. */
13564 hmR0VmxSetPendingXcptUD(pVCpu);
13565 return VINF_SUCCESS;
13566}
13567
13568
13569/**
13570 * VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Unconditional VM-exit.
13571 */
13572HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13573{
13574 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13575
13576 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13577 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13578 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13579 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13580 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13581 AssertRCReturn(rc, rc);
13582
13583 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13584
13585 VMXVEXITINFO ExitInfo;
13586 RT_ZERO(ExitInfo);
13587 ExitInfo.uReason = pVmxTransient->uExitReason;
13588 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13589 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13590 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13591 if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
13592 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13593
13594 VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
13595 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13596 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13597 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13598 {
13599 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13600 rcStrict = VINF_SUCCESS;
13601 }
13602 return rcStrict;
13603}
13604
13605
13606/**
13607 * VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
13608 */
13609HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13610{
13611 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13612
13613 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13614 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13615 AssertRCReturn(rc, rc);
13616
13617 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13618
13619 VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
13620 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13621 {
13622 /* VMXOFF on success changes the internal hwvirt state but not anything that's visible to the guest. */
13623 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
13624 }
13625 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13626 {
13627 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13628 rcStrict = VINF_SUCCESS;
13629 }
13630 return rcStrict;
13631}
13632
13633
13634/**
13635 * VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
13636 */
13637HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13638{
13639 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13640
13641 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13642 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13643 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13644 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13645 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13646 AssertRCReturn(rc, rc);
13647
13648 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13649
13650 VMXVEXITINFO ExitInfo;
13651 RT_ZERO(ExitInfo);
13652 ExitInfo.uReason = pVmxTransient->uExitReason;
13653 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13654 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13655 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13656 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13657
13658 VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
13659 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13660 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13661 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13662 {
13663 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13664 rcStrict = VINF_SUCCESS;
13665 }
13666 return rcStrict;
13667}
13668
13669/** @} */
13670#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13671
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette