VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 75857

Last change on this file since 75857 was 75857, checked in by vboxsync, 6 years ago

VMM/HMVMXR0: Build fix.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 576.7 KB
Line 
1/* $Id: HMVMXR0.cpp 75857 2018-12-01 11:37:58Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/x86.h>
25#include <iprt/asm-amd64-x86.h>
26#include <iprt/thread.h>
27
28#include <VBox/vmm/pdmapi.h>
29#include <VBox/vmm/dbgf.h>
30#include <VBox/vmm/iem.h>
31#include <VBox/vmm/iom.h>
32#include <VBox/vmm/selm.h>
33#include <VBox/vmm/tm.h>
34#include <VBox/vmm/em.h>
35#include <VBox/vmm/gim.h>
36#include <VBox/vmm/apic.h>
37#ifdef VBOX_WITH_REM
38# include <VBox/vmm/rem.h>
39#endif
40#include "HMInternal.h"
41#include <VBox/vmm/vm.h>
42#include "HMVMXR0.h"
43#include "dtrace/VBoxVMM.h"
44
45#ifdef DEBUG_ramshankar
46# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
47# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
49# define HMVMX_ALWAYS_CHECK_GUEST_STATE
50# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
51# define HMVMX_ALWAYS_TRAP_PF
52# define HMVMX_ALWAYS_FLUSH_TLB
53# define HMVMX_ALWAYS_SWAP_EFER
54#endif
55
56
57/*********************************************************************************************************************************
58* Defined Constants And Macros *
59*********************************************************************************************************************************/
60/** Use the function table. */
61#define HMVMX_USE_FUNCTION_TABLE
62
63/** Determine which tagged-TLB flush handler to use. */
64#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
65#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
66#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
67#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
68
69/** @name HMVMX_READ_XXX
70 * Flags to skip redundant reads of some common VMCS fields that are not part of
71 * the guest-CPU or VCPU state but are needed while handling VM-exits.
72 */
73#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
74#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
75#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
76#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
77#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
78#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
79#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
80#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
81/** @} */
82
83/**
84 * States of the VMCS.
85 *
86 * This does not reflect all possible VMCS states but currently only those
87 * needed for maintaining the VMCS consistently even when thread-context hooks
88 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
89 */
90#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
91#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
92#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
93
94/**
95 * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
96 * guest using hardware-assisted VMX.
97 *
98 * This excludes state like GPRs (other than RSP) which are always are
99 * swapped and restored across the world-switch and also registers like EFER,
100 * MSR which cannot be modified by the guest without causing a VM-exit.
101 */
102#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
103 | CPUMCTX_EXTRN_RFLAGS \
104 | CPUMCTX_EXTRN_RSP \
105 | CPUMCTX_EXTRN_SREG_MASK \
106 | CPUMCTX_EXTRN_TABLE_MASK \
107 | CPUMCTX_EXTRN_KERNEL_GS_BASE \
108 | CPUMCTX_EXTRN_SYSCALL_MSRS \
109 | CPUMCTX_EXTRN_SYSENTER_MSRS \
110 | CPUMCTX_EXTRN_TSC_AUX \
111 | CPUMCTX_EXTRN_OTHER_MSRS \
112 | CPUMCTX_EXTRN_CR0 \
113 | CPUMCTX_EXTRN_CR3 \
114 | CPUMCTX_EXTRN_CR4 \
115 | CPUMCTX_EXTRN_DR7 \
116 | CPUMCTX_EXTRN_HM_VMX_MASK)
117
118/**
119 * Exception bitmap mask for real-mode guests (real-on-v86).
120 *
121 * We need to intercept all exceptions manually except:
122 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
123 * due to bugs in Intel CPUs.
124 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
125 * support.
126 */
127#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
128 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
129 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
130 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
131 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
132 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
133 | RT_BIT(X86_XCPT_XF))
134
135/** Maximum VM-instruction error number. */
136#define HMVMX_INSTR_ERROR_MAX 28
137
138/** Profiling macro. */
139#ifdef HM_PROFILE_EXIT_DISPATCH
140# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
141# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
142#else
143# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
144# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
145#endif
146
147/** Assert that preemption is disabled or covered by thread-context hooks. */
148#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
149 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
150
151/** Assert that we haven't migrated CPUs when thread-context hooks are not
152 * used. */
153#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
154 || (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
155 ("Illegal migration! Entered on CPU %u Current %u\n", \
156 (a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
157
158/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
159 * context. */
160#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
161 ("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
162 (a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
163
164/** Macro for importing guest state from the VMCS back into CPUMCTX (intended to be
165 * used only from VM-exit handlers). */
166#define HMVMX_CPUMCTX_IMPORT_STATE(a_pVCpu, a_fWhat) (hmR0VmxImportGuestState((a_pVCpu), (a_fWhat)))
167
168/** Helper macro for VM-exit handlers called unexpectedly. */
169#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
170 do { \
171 (a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
172 return VERR_VMX_UNEXPECTED_EXIT; \
173 } while (0)
174
175/** Macro for importing segment registers to the VMCS from the guest-CPU context. */
176#ifdef VMX_USE_CACHED_VMCS_ACCESSES
177# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
178 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
179 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
180#else
181# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
182 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
183 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
184#endif
185
186/** Macro for exporting segment registers to the VMCS from the guest-CPU context. */
187#define HMVMX_EXPORT_SREG(Sel, a_pCtxSelReg) \
188 hmR0VmxExportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
189 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
190
191#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
192/** Macro that does the necessary privilege checks and intercepted VM-exits for
193 * guests that attempted to execute a VMX instruction. */
194# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
195 do \
196 { \
197 VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
198 if (rcStrictTmp == VINF_SUCCESS) \
199 { /* likely */ } \
200 else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
201 { \
202 Assert((a_pVCpu)->hm.s.Event.fPending); \
203 Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
204 return VINF_SUCCESS; \
205 } \
206 else \
207 { \
208 int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
209 AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
210 } \
211 } while (0)
212
213/** Macro that decodes a memory operand for an instruction VM-exit. */
214# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
215 do \
216 { \
217 VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
218 (a_pGCPtrEffAddr)); \
219 if (rcStrictTmp == VINF_SUCCESS) \
220 { /* likely */ } \
221 else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
222 { \
223 uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
224 Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
225 return VINF_SUCCESS; \
226 } \
227 else \
228 { \
229 Log4Func(("hmR0VmxCheckExitDueToVmxInstr failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
230 return rcStrictTmp; \
231 } \
232 } while (0)
233
234#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
235
236
237/*********************************************************************************************************************************
238* Structures and Typedefs *
239*********************************************************************************************************************************/
240/**
241 * VMX transient state.
242 *
243 * A state structure for holding miscellaneous information across
244 * VMX non-root operation and restored after the transition.
245 */
246typedef struct VMXTRANSIENT
247{
248 /** The host's rflags/eflags. */
249 RTCCUINTREG fEFlags;
250#if HC_ARCH_BITS == 32
251 uint32_t u32Alignment0;
252#endif
253 /** The guest's TPR value used for TPR shadowing. */
254 uint8_t u8GuestTpr;
255 /** Alignment. */
256 uint8_t abAlignment0[7];
257
258 /** The basic VM-exit reason. */
259 uint16_t uExitReason;
260 /** Alignment. */
261 uint16_t u16Alignment0;
262 /** The VM-exit interruption error code. */
263 uint32_t uExitIntErrorCode;
264 /** The VM-exit exit code qualification. */
265 uint64_t uExitQual;
266 /** The Guest-linear address. */
267 uint64_t uGuestLinearAddr;
268
269 /** The VM-exit interruption-information field. */
270 uint32_t uExitIntInfo;
271 /** The VM-exit instruction-length field. */
272 uint32_t cbInstr;
273 /** The VM-exit instruction-information field. */
274 VMXEXITINSTRINFO ExitInstrInfo;
275 /** Whether the VM-entry failed or not. */
276 bool fVMEntryFailed;
277 /** Alignment. */
278 uint8_t abAlignment1[3];
279
280 /** The VM-entry interruption-information field. */
281 uint32_t uEntryIntInfo;
282 /** The VM-entry exception error code field. */
283 uint32_t uEntryXcptErrorCode;
284 /** The VM-entry instruction length field. */
285 uint32_t cbEntryInstr;
286
287 /** IDT-vectoring information field. */
288 uint32_t uIdtVectoringInfo;
289 /** IDT-vectoring error code. */
290 uint32_t uIdtVectoringErrorCode;
291
292 /** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
293 uint32_t fVmcsFieldsRead;
294
295 /** Whether the guest debug state was active at the time of VM-exit. */
296 bool fWasGuestDebugStateActive;
297 /** Whether the hyper debug state was active at the time of VM-exit. */
298 bool fWasHyperDebugStateActive;
299 /** Whether TSC-offsetting should be setup before VM-entry. */
300 bool fUpdateTscOffsettingAndPreemptTimer;
301 /** Whether the VM-exit was caused by a page-fault during delivery of a
302 * contributory exception or a page-fault. */
303 bool fVectoringDoublePF;
304 /** Whether the VM-exit was caused by a page-fault during delivery of an
305 * external interrupt or NMI. */
306 bool fVectoringPF;
307} VMXTRANSIENT;
308AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
309AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
310AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
311AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
312AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
313/** Pointer to VMX transient state. */
314typedef VMXTRANSIENT *PVMXTRANSIENT;
315
316/**
317 * Memory operand read or write access.
318 */
319typedef enum VMXMEMACCESS
320{
321 VMXMEMACCESS_READ = 0,
322 VMXMEMACCESS_WRITE = 1
323} VMXMEMACCESS;
324
325/**
326 * VMX VM-exit handler.
327 *
328 * @returns Strict VBox status code (i.e. informational status codes too).
329 * @param pVCpu The cross context virtual CPU structure.
330 * @param pVmxTransient Pointer to the VMX-transient structure.
331 */
332#ifndef HMVMX_USE_FUNCTION_TABLE
333typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
334#else
335typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
336/** Pointer to VM-exit handler. */
337typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
338#endif
339
340/**
341 * VMX VM-exit handler, non-strict status code.
342 *
343 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
344 *
345 * @returns VBox status code, no informational status code returned.
346 * @param pVCpu The cross context virtual CPU structure.
347 * @param pVmxTransient Pointer to the VMX-transient structure.
348 *
349 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
350 * use of that status code will be replaced with VINF_EM_SOMETHING
351 * later when switching over to IEM.
352 */
353#ifndef HMVMX_USE_FUNCTION_TABLE
354typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
355#else
356typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
357#endif
358
359
360/*********************************************************************************************************************************
361* Internal Functions *
362*********************************************************************************************************************************/
363static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush);
364static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr);
365static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
366static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
367static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
368 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
369#if HC_ARCH_BITS == 32
370static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
371#endif
372#ifndef HMVMX_USE_FUNCTION_TABLE
373DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
374# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
375# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
376#else
377# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
378# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
379#endif
380
381/** @name VM-exit handlers.
382 * @{
383 */
384static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
385static FNVMXEXITHANDLER hmR0VmxExitExtInt;
386static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
387static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
388static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
389static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
390static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
391static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
392static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
393static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
394static FNVMXEXITHANDLER hmR0VmxExitCpuid;
395static FNVMXEXITHANDLER hmR0VmxExitGetsec;
396static FNVMXEXITHANDLER hmR0VmxExitHlt;
397static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
398static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
399static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
400static FNVMXEXITHANDLER hmR0VmxExitVmcall;
401#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
402static FNVMXEXITHANDLER hmR0VmxExitVmclear;
403static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
404static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
405static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
406static FNVMXEXITHANDLER hmR0VmxExitVmread;
407static FNVMXEXITHANDLER hmR0VmxExitVmresume;
408static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
409static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
410static FNVMXEXITHANDLER hmR0VmxExitVmxon;
411#endif
412static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
413static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
414static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
415static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
416static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
417static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
418static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
419static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
420static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
421static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
422static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
423static FNVMXEXITHANDLER hmR0VmxExitMwait;
424static FNVMXEXITHANDLER hmR0VmxExitMtf;
425static FNVMXEXITHANDLER hmR0VmxExitMonitor;
426static FNVMXEXITHANDLER hmR0VmxExitPause;
427static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
428static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
429static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
430static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
431static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
432static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
433static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
434static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
435static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
436static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
437static FNVMXEXITHANDLER hmR0VmxExitRdrand;
438static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
439/** @} */
440
441static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
442static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
443static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
444static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
445static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
446static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
447static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
448static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu);
449
450
451/*********************************************************************************************************************************
452* Global Variables *
453*********************************************************************************************************************************/
454#ifdef HMVMX_USE_FUNCTION_TABLE
455
456/**
457 * VMX_EXIT dispatch table.
458 */
459static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
460{
461 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
462 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
463 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
464 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
465 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
466 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
467 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
468 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
469 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
470 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
471 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
472 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
473 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
474 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
475 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
476 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
477 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
478 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
479 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
480#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
481 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
482 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
483 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
484 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
485 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
486 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
487 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
488 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
489 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
490#else
491 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
492 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
493 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
494 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
495 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
496 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
497 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
498 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
499 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
500#endif
501 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
502 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
503 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
504 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
505 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
506 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
507 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
508 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
509 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
510 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
511 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
512 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
513 /* 40 UNDEFINED */ hmR0VmxExitPause,
514 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
515 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
516 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
517 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
518 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
519 /* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitXdtrAccess,
520 /* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitXdtrAccess,
521 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
522 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
523 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
524 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
525 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
526 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
527 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
528 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
529 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
530 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
531 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
532 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
533 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
534 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
535 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
536 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
537 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
538};
539#endif /* HMVMX_USE_FUNCTION_TABLE */
540
541#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
542static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
543{
544 /* 0 */ "(Not Used)",
545 /* 1 */ "VMCALL executed in VMX root operation.",
546 /* 2 */ "VMCLEAR with invalid physical address.",
547 /* 3 */ "VMCLEAR with VMXON pointer.",
548 /* 4 */ "VMLAUNCH with non-clear VMCS.",
549 /* 5 */ "VMRESUME with non-launched VMCS.",
550 /* 6 */ "VMRESUME after VMXOFF",
551 /* 7 */ "VM-entry with invalid control fields.",
552 /* 8 */ "VM-entry with invalid host state fields.",
553 /* 9 */ "VMPTRLD with invalid physical address.",
554 /* 10 */ "VMPTRLD with VMXON pointer.",
555 /* 11 */ "VMPTRLD with incorrect revision identifier.",
556 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
557 /* 13 */ "VMWRITE to read-only VMCS component.",
558 /* 14 */ "(Not Used)",
559 /* 15 */ "VMXON executed in VMX root operation.",
560 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
561 /* 17 */ "VM-entry with non-launched executing VMCS.",
562 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
563 /* 19 */ "VMCALL with non-clear VMCS.",
564 /* 20 */ "VMCALL with invalid VM-exit control fields.",
565 /* 21 */ "(Not Used)",
566 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
567 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
568 /* 24 */ "VMCALL with invalid SMM-monitor features.",
569 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
570 /* 26 */ "VM-entry with events blocked by MOV SS.",
571 /* 27 */ "(Not Used)",
572 /* 28 */ "Invalid operand to INVEPT/INVVPID."
573};
574#endif /* VBOX_STRICT */
575
576
577/**
578 * Updates the VM's last error record.
579 *
580 * If there was a VMX instruction error, reads the error data from the VMCS and
581 * updates VCPU's last error record as well.
582 *
583 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
584 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
585 * VERR_VMX_INVALID_VMCS_FIELD.
586 * @param rc The error code.
587 */
588static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
589{
590 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
591 || rc == VERR_VMX_UNABLE_TO_START_VM)
592 {
593 AssertPtrReturnVoid(pVCpu);
594 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
595 }
596 pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
597}
598
599
600/**
601 * Reads the VM-entry interruption-information field from the VMCS into the VMX
602 * transient structure.
603 *
604 * @returns VBox status code.
605 * @param pVmxTransient Pointer to the VMX transient structure.
606 *
607 * @remarks No-long-jump zone!!!
608 */
609DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
610{
611 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
612 AssertRCReturn(rc, rc);
613 return VINF_SUCCESS;
614}
615
616#ifdef VBOX_STRICT
617/**
618 * Reads the VM-entry exception error code field from the VMCS into
619 * the VMX transient structure.
620 *
621 * @returns VBox status code.
622 * @param pVmxTransient Pointer to the VMX transient structure.
623 *
624 * @remarks No-long-jump zone!!!
625 */
626DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
627{
628 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
629 AssertRCReturn(rc, rc);
630 return VINF_SUCCESS;
631}
632
633
634/**
635 * Reads the VM-entry exception error code field from the VMCS into
636 * the VMX transient structure.
637 *
638 * @returns VBox status code.
639 * @param pVmxTransient Pointer to the VMX transient structure.
640 *
641 * @remarks No-long-jump zone!!!
642 */
643DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
644{
645 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
646 AssertRCReturn(rc, rc);
647 return VINF_SUCCESS;
648}
649#endif /* VBOX_STRICT */
650
651
652/**
653 * Reads the VM-exit interruption-information field from the VMCS into the VMX
654 * transient structure.
655 *
656 * @returns VBox status code.
657 * @param pVmxTransient Pointer to the VMX transient structure.
658 */
659DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
660{
661 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
662 {
663 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
664 AssertRCReturn(rc,rc);
665 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_INFO;
666 }
667 return VINF_SUCCESS;
668}
669
670
671/**
672 * Reads the VM-exit interruption error code from the VMCS into the VMX
673 * transient structure.
674 *
675 * @returns VBox status code.
676 * @param pVmxTransient Pointer to the VMX transient structure.
677 */
678DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
679{
680 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
681 {
682 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
683 AssertRCReturn(rc, rc);
684 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
685 }
686 return VINF_SUCCESS;
687}
688
689
690/**
691 * Reads the VM-exit instruction length field from the VMCS into the VMX
692 * transient structure.
693 *
694 * @returns VBox status code.
695 * @param pVmxTransient Pointer to the VMX transient structure.
696 */
697DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
698{
699 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
700 {
701 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
702 AssertRCReturn(rc, rc);
703 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_LEN;
704 }
705 return VINF_SUCCESS;
706}
707
708
709/**
710 * Reads the VM-exit instruction-information field from the VMCS into
711 * the VMX transient structure.
712 *
713 * @returns VBox status code.
714 * @param pVmxTransient Pointer to the VMX transient structure.
715 */
716DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
717{
718 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
719 {
720 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
721 AssertRCReturn(rc, rc);
722 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_INFO;
723 }
724 return VINF_SUCCESS;
725}
726
727
728/**
729 * Reads the VM-exit Qualification from the VMCS into the VMX transient structure.
730 *
731 * @returns VBox status code.
732 * @param pVCpu The cross context virtual CPU structure of the
733 * calling EMT. (Required for the VMCS cache case.)
734 * @param pVmxTransient Pointer to the VMX transient structure.
735 */
736DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
737{
738 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
739 {
740 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
741 AssertRCReturn(rc, rc);
742 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION;
743 }
744 return VINF_SUCCESS;
745}
746
747
748/**
749 * Reads the Guest-linear address from the VMCS into the VMX transient structure.
750 *
751 * @returns VBox status code.
752 * @param pVCpu The cross context virtual CPU structure of the
753 * calling EMT. (Required for the VMCS cache case.)
754 * @param pVmxTransient Pointer to the VMX transient structure.
755 */
756DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
757{
758 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
759 {
760 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
761 AssertRCReturn(rc, rc);
762 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_GUEST_LINEAR_ADDR;
763 }
764 return VINF_SUCCESS;
765}
766
767
768/**
769 * Reads the IDT-vectoring information field from the VMCS into the VMX
770 * transient structure.
771 *
772 * @returns VBox status code.
773 * @param pVmxTransient Pointer to the VMX transient structure.
774 *
775 * @remarks No-long-jump zone!!!
776 */
777DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
778{
779 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
780 {
781 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
782 AssertRCReturn(rc, rc);
783 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_INFO;
784 }
785 return VINF_SUCCESS;
786}
787
788
789/**
790 * Reads the IDT-vectoring error code from the VMCS into the VMX
791 * transient structure.
792 *
793 * @returns VBox status code.
794 * @param pVmxTransient Pointer to the VMX transient structure.
795 */
796DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
797{
798 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
799 {
800 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
801 AssertRCReturn(rc, rc);
802 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
803 }
804 return VINF_SUCCESS;
805}
806
807
808/**
809 * Enters VMX root mode operation on the current CPU.
810 *
811 * @returns VBox status code.
812 * @param pVM The cross context VM structure. Can be
813 * NULL, after a resume.
814 * @param HCPhysCpuPage Physical address of the VMXON region.
815 * @param pvCpuPage Pointer to the VMXON region.
816 */
817static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
818{
819 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
820 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
821 Assert(pvCpuPage);
822 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
823
824 if (pVM)
825 {
826 /* Write the VMCS revision dword to the VMXON region. */
827 *(uint32_t *)pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
828 }
829
830 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
831 RTCCUINTREG fEFlags = ASMIntDisableFlags();
832
833 /* Enable the VMX bit in CR4 if necessary. */
834 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
835
836 /* Enter VMX root mode. */
837 int rc = VMXEnable(HCPhysCpuPage);
838 if (RT_FAILURE(rc))
839 {
840 if (!(uOldCr4 & X86_CR4_VMXE))
841 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
842
843 if (pVM)
844 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
845 }
846
847 /* Restore interrupts. */
848 ASMSetFlags(fEFlags);
849 return rc;
850}
851
852
853/**
854 * Exits VMX root mode operation on the current CPU.
855 *
856 * @returns VBox status code.
857 */
858static int hmR0VmxLeaveRootMode(void)
859{
860 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
861
862 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
863 RTCCUINTREG fEFlags = ASMIntDisableFlags();
864
865 /* If we're for some reason not in VMX root mode, then don't leave it. */
866 RTCCUINTREG uHostCR4 = ASMGetCR4();
867
868 int rc;
869 if (uHostCR4 & X86_CR4_VMXE)
870 {
871 /* Exit VMX root mode and clear the VMX bit in CR4. */
872 VMXDisable();
873 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
874 rc = VINF_SUCCESS;
875 }
876 else
877 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
878
879 /* Restore interrupts. */
880 ASMSetFlags(fEFlags);
881 return rc;
882}
883
884
885/**
886 * Allocates and maps one physically contiguous page. The allocated page is
887 * zero'd out. (Used by various VT-x structures).
888 *
889 * @returns IPRT status code.
890 * @param pMemObj Pointer to the ring-0 memory object.
891 * @param ppVirt Where to store the virtual address of the
892 * allocation.
893 * @param pHCPhys Where to store the physical address of the
894 * allocation.
895 */
896static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
897{
898 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
899 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
900 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
901
902 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
903 if (RT_FAILURE(rc))
904 return rc;
905 *ppVirt = RTR0MemObjAddress(*pMemObj);
906 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
907 ASMMemZero32(*ppVirt, PAGE_SIZE);
908 return VINF_SUCCESS;
909}
910
911
912/**
913 * Frees and unmaps an allocated physical page.
914 *
915 * @param pMemObj Pointer to the ring-0 memory object.
916 * @param ppVirt Where to re-initialize the virtual address of
917 * allocation as 0.
918 * @param pHCPhys Where to re-initialize the physical address of the
919 * allocation as 0.
920 */
921static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
922{
923 AssertPtr(pMemObj);
924 AssertPtr(ppVirt);
925 AssertPtr(pHCPhys);
926 if (*pMemObj != NIL_RTR0MEMOBJ)
927 {
928 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
929 AssertRC(rc);
930 *pMemObj = NIL_RTR0MEMOBJ;
931 *ppVirt = 0;
932 *pHCPhys = 0;
933 }
934}
935
936
937/**
938 * Worker function to free VT-x related structures.
939 *
940 * @returns IPRT status code.
941 * @param pVM The cross context VM structure.
942 */
943static void hmR0VmxStructsFree(PVM pVM)
944{
945 for (VMCPUID i = 0; i < pVM->cCpus; i++)
946 {
947 PVMCPU pVCpu = &pVM->aCpus[i];
948 AssertPtr(pVCpu);
949
950 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
951 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
952
953 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
954 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
955
956 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
957 }
958
959 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
960#ifdef VBOX_WITH_CRASHDUMP_MAGIC
961 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
962#endif
963}
964
965
966/**
967 * Worker function to allocate VT-x related VM structures.
968 *
969 * @returns IPRT status code.
970 * @param pVM The cross context VM structure.
971 */
972static int hmR0VmxStructsAlloc(PVM pVM)
973{
974 /*
975 * Initialize members up-front so we can cleanup properly on allocation failure.
976 */
977#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
978 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
979 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
980 pVM->hm.s.vmx.HCPhys##a_Name = 0;
981
982#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
983 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
984 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
985 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
986
987#ifdef VBOX_WITH_CRASHDUMP_MAGIC
988 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
989#endif
990 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
991
992 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
993 for (VMCPUID i = 0; i < pVM->cCpus; i++)
994 {
995 PVMCPU pVCpu = &pVM->aCpus[i];
996 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
997 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
998 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
999 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
1000 }
1001#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
1002#undef VMXLOCAL_INIT_VM_MEMOBJ
1003
1004 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
1005 AssertReturnStmt(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE) <= PAGE_SIZE,
1006 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
1007 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
1008
1009 /*
1010 * Allocate all the VT-x structures.
1011 */
1012 int rc = VINF_SUCCESS;
1013#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1014 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1015 if (RT_FAILURE(rc))
1016 goto cleanup;
1017 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1018 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1019#endif
1020
1021 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1022 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1023 {
1024 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1025 &pVM->hm.s.vmx.HCPhysApicAccess);
1026 if (RT_FAILURE(rc))
1027 goto cleanup;
1028 }
1029
1030 /*
1031 * Initialize per-VCPU VT-x structures.
1032 */
1033 for (VMCPUID i = 0; i < pVM->cCpus; i++)
1034 {
1035 PVMCPU pVCpu = &pVM->aCpus[i];
1036 AssertPtr(pVCpu);
1037
1038 /* Allocate the VM control structure (VMCS). */
1039 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1040 if (RT_FAILURE(rc))
1041 goto cleanup;
1042
1043 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1044 if ( PDMHasApic(pVM)
1045 && (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1046 {
1047 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1048 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1049 if (RT_FAILURE(rc))
1050 goto cleanup;
1051 }
1052
1053 /*
1054 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1055 * transparent accesses of specific MSRs.
1056 *
1057 * If the condition for enabling MSR bitmaps changes here, don't forget to
1058 * update HMAreMsrBitmapsAvailable().
1059 */
1060 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1061 {
1062 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1063 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1064 if (RT_FAILURE(rc))
1065 goto cleanup;
1066 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1067 }
1068
1069 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1070 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1071 if (RT_FAILURE(rc))
1072 goto cleanup;
1073
1074 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1075 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1076 if (RT_FAILURE(rc))
1077 goto cleanup;
1078 }
1079
1080 return VINF_SUCCESS;
1081
1082cleanup:
1083 hmR0VmxStructsFree(pVM);
1084 return rc;
1085}
1086
1087
1088/**
1089 * Does global VT-x initialization (called during module initialization).
1090 *
1091 * @returns VBox status code.
1092 */
1093VMMR0DECL(int) VMXR0GlobalInit(void)
1094{
1095#ifdef HMVMX_USE_FUNCTION_TABLE
1096 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1097# ifdef VBOX_STRICT
1098 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1099 Assert(g_apfnVMExitHandlers[i]);
1100# endif
1101#endif
1102 return VINF_SUCCESS;
1103}
1104
1105
1106/**
1107 * Does global VT-x termination (called during module termination).
1108 */
1109VMMR0DECL(void) VMXR0GlobalTerm()
1110{
1111 /* Nothing to do currently. */
1112}
1113
1114
1115/**
1116 * Sets up and activates VT-x on the current CPU.
1117 *
1118 * @returns VBox status code.
1119 * @param pHostCpu Pointer to the global CPU info struct.
1120 * @param pVM The cross context VM structure. Can be
1121 * NULL after a host resume operation.
1122 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1123 * fEnabledByHost is @c true).
1124 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1125 * @a fEnabledByHost is @c true).
1126 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1127 * enable VT-x on the host.
1128 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1129 */
1130VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1131 void *pvMsrs)
1132{
1133 Assert(pHostCpu);
1134 Assert(pvMsrs);
1135 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1136
1137 /* Enable VT-x if it's not already enabled by the host. */
1138 if (!fEnabledByHost)
1139 {
1140 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1141 if (RT_FAILURE(rc))
1142 return rc;
1143 }
1144
1145 /*
1146 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1147 * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1148 * invalidated when flushing by VPID.
1149 */
1150 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1151 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1152 {
1153 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
1154 pHostCpu->fFlushAsidBeforeUse = false;
1155 }
1156 else
1157 pHostCpu->fFlushAsidBeforeUse = true;
1158
1159 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1160 ++pHostCpu->cTlbFlushes;
1161
1162 return VINF_SUCCESS;
1163}
1164
1165
1166/**
1167 * Deactivates VT-x on the current CPU.
1168 *
1169 * @returns VBox status code.
1170 * @param pHostCpu Pointer to the global CPU info struct.
1171 * @param pvCpuPage Pointer to the VMXON region.
1172 * @param HCPhysCpuPage Physical address of the VMXON region.
1173 *
1174 * @remarks This function should never be called when SUPR0EnableVTx() or
1175 * similar was used to enable VT-x on the host.
1176 */
1177VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1178{
1179 RT_NOREF3(pHostCpu, pvCpuPage, HCPhysCpuPage);
1180
1181 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1182 return hmR0VmxLeaveRootMode();
1183}
1184
1185
1186/**
1187 * Sets the permission bits for the specified MSR in the MSR bitmap.
1188 *
1189 * @param pVCpu The cross context virtual CPU structure.
1190 * @param uMsr The MSR value.
1191 * @param enmRead Whether reading this MSR causes a VM-exit.
1192 * @param enmWrite Whether writing this MSR causes a VM-exit.
1193 */
1194static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1195{
1196 int32_t iBit;
1197 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1198
1199 /*
1200 * MSR Layout:
1201 * Byte index MSR range Interpreted as
1202 * 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
1203 * 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
1204 * 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
1205 * 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
1206 *
1207 * A bit corresponding to an MSR within the above range causes a VM-exit
1208 * if the bit is 1 on executions of RDMSR/WRMSR.
1209 *
1210 * If an MSR falls out of the MSR range, it always cause a VM-exit.
1211 *
1212 * See Intel spec. 24.6.9 "MSR-Bitmap Address".
1213 */
1214 if (uMsr <= 0x00001fff)
1215 iBit = uMsr;
1216 else if (uMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1217 {
1218 iBit = uMsr - UINT32_C(0xc0000000);
1219 pbMsrBitmap += 0x400;
1220 }
1221 else
1222 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1223
1224 Assert(iBit <= 0x1fff);
1225 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1226 ASMBitSet(pbMsrBitmap, iBit);
1227 else
1228 ASMBitClear(pbMsrBitmap, iBit);
1229
1230 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1231 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1232 else
1233 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1234}
1235
1236
1237/**
1238 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1239 * area.
1240 *
1241 * @returns VBox status code.
1242 * @param pVCpu The cross context virtual CPU structure.
1243 * @param cMsrs The number of MSRs.
1244 */
1245static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1246{
1247 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1248 uint64_t const uVmxMiscMsr = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc;
1249 uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(uVmxMiscMsr);
1250 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1251 {
1252 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1253 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1254 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1255 }
1256
1257 /* Update number of guest MSRs to load/store across the world-switch. */
1258 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1259 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1260
1261 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1262 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1263 AssertRCReturn(rc, rc);
1264
1265 /* Update the VCPU's copy of the MSR count. */
1266 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1267
1268 return VINF_SUCCESS;
1269}
1270
1271
1272/**
1273 * Adds a new (or updates the value of an existing) guest/host MSR
1274 * pair to be swapped during the world-switch as part of the
1275 * auto-load/store MSR area in the VMCS.
1276 *
1277 * @returns VBox status code.
1278 * @param pVCpu The cross context virtual CPU structure.
1279 * @param uMsr The MSR.
1280 * @param uGuestMsrValue Value of the guest MSR.
1281 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1282 * necessary.
1283 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1284 * its value was updated. Optional, can be NULL.
1285 */
1286static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1287 bool *pfAddedAndUpdated)
1288{
1289 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1290 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1291 uint32_t i;
1292 for (i = 0; i < cMsrs; i++)
1293 {
1294 if (pGuestMsr->u32Msr == uMsr)
1295 break;
1296 pGuestMsr++;
1297 }
1298
1299 bool fAdded = false;
1300 if (i == cMsrs)
1301 {
1302 ++cMsrs;
1303 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1304 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1305
1306 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1307 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1308 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1309
1310 fAdded = true;
1311 }
1312
1313 /* Update the MSR values in the auto-load/store MSR area. */
1314 pGuestMsr->u32Msr = uMsr;
1315 pGuestMsr->u64Value = uGuestMsrValue;
1316
1317 /* Create/update the MSR slot in the host MSR area. */
1318 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1319 pHostMsr += i;
1320 pHostMsr->u32Msr = uMsr;
1321
1322 /*
1323 * Update the host MSR only when requested by the caller AND when we're
1324 * adding it to the auto-load/store area. Otherwise, it would have been
1325 * updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1326 */
1327 bool fUpdatedMsrValue = false;
1328 if ( fAdded
1329 && fUpdateHostMsr)
1330 {
1331 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1332 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1333 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1334 fUpdatedMsrValue = true;
1335 }
1336
1337 if (pfAddedAndUpdated)
1338 *pfAddedAndUpdated = fUpdatedMsrValue;
1339 return VINF_SUCCESS;
1340}
1341
1342
1343/**
1344 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1345 * auto-load/store MSR area in the VMCS.
1346 *
1347 * @returns VBox status code.
1348 * @param pVCpu The cross context virtual CPU structure.
1349 * @param uMsr The MSR.
1350 */
1351static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1352{
1353 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1354 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1355 for (uint32_t i = 0; i < cMsrs; i++)
1356 {
1357 /* Find the MSR. */
1358 if (pGuestMsr->u32Msr == uMsr)
1359 {
1360 /* If it's the last MSR, simply reduce the count. */
1361 if (i == cMsrs - 1)
1362 {
1363 --cMsrs;
1364 break;
1365 }
1366
1367 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1368 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1369 pLastGuestMsr += cMsrs - 1;
1370 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1371 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1372
1373 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1374 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1375 pLastHostMsr += cMsrs - 1;
1376 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1377 pHostMsr->u64Value = pLastHostMsr->u64Value;
1378 --cMsrs;
1379 break;
1380 }
1381 pGuestMsr++;
1382 }
1383
1384 /* Update the VMCS if the count changed (meaning the MSR was found). */
1385 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1386 {
1387 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1388 AssertRCReturn(rc, rc);
1389
1390 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1391 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1392 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1393
1394 Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1395 return VINF_SUCCESS;
1396 }
1397
1398 return VERR_NOT_FOUND;
1399}
1400
1401
1402/**
1403 * Checks if the specified guest MSR is part of the auto-load/store area in
1404 * the VMCS.
1405 *
1406 * @returns true if found, false otherwise.
1407 * @param pVCpu The cross context virtual CPU structure.
1408 * @param uMsr The MSR to find.
1409 */
1410static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1411{
1412 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1413 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1414
1415 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1416 {
1417 if (pGuestMsr->u32Msr == uMsr)
1418 return true;
1419 }
1420 return false;
1421}
1422
1423
1424/**
1425 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1426 *
1427 * @param pVCpu The cross context virtual CPU structure.
1428 *
1429 * @remarks No-long-jump zone!!!
1430 */
1431static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1432{
1433 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1434 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1435 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1436 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1437
1438 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1439 {
1440 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1441
1442 /*
1443 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1444 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1445 */
1446 if (pHostMsr->u32Msr == MSR_K6_EFER)
1447 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1448 else
1449 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1450 }
1451
1452 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1453}
1454
1455
1456/**
1457 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1458 * perform lazy restoration of the host MSRs while leaving VT-x.
1459 *
1460 * @param pVCpu The cross context virtual CPU structure.
1461 *
1462 * @remarks No-long-jump zone!!!
1463 */
1464static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1465{
1466 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1467
1468 /*
1469 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1470 */
1471 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1472 {
1473 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1474#if HC_ARCH_BITS == 64
1475 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1476 {
1477 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1478 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1479 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1480 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1481 }
1482#endif
1483 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1484 }
1485}
1486
1487
1488/**
1489 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1490 * lazily while leaving VT-x.
1491 *
1492 * @returns true if it does, false otherwise.
1493 * @param pVCpu The cross context virtual CPU structure.
1494 * @param uMsr The MSR to check.
1495 */
1496static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1497{
1498 NOREF(pVCpu);
1499#if HC_ARCH_BITS == 64
1500 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1501 {
1502 switch (uMsr)
1503 {
1504 case MSR_K8_LSTAR:
1505 case MSR_K6_STAR:
1506 case MSR_K8_SF_MASK:
1507 case MSR_K8_KERNEL_GS_BASE:
1508 return true;
1509 }
1510 }
1511#else
1512 RT_NOREF(pVCpu, uMsr);
1513#endif
1514 return false;
1515}
1516
1517
1518/**
1519 * Loads a set of guests MSRs to allow read/passthru to the guest.
1520 *
1521 * The name of this function is slightly confusing. This function does NOT
1522 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1523 * common prefix for functions dealing with "lazy restoration" of the shared
1524 * MSRs.
1525 *
1526 * @param pVCpu The cross context virtual CPU structure.
1527 *
1528 * @remarks No-long-jump zone!!!
1529 */
1530static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1531{
1532 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1533 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1534
1535 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1536#if HC_ARCH_BITS == 64
1537 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1538 {
1539 /*
1540 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1541 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1542 * we can skip a few MSR writes.
1543 *
1544 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1545 * guest MSR values in the guest-CPU context might be different to what's currently
1546 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1547 * CPU, see @bugref{8728}.
1548 */
1549 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1550 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1551 && pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1552 && pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1553 && pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1554 && pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1555 {
1556#ifdef VBOX_STRICT
1557 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1558 Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1559 Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1560 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
1561#endif
1562 }
1563 else
1564 {
1565 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1566 ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
1567 ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
1568 ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
1569 }
1570 }
1571#endif
1572 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1573}
1574
1575
1576/**
1577 * Performs lazy restoration of the set of host MSRs if they were previously
1578 * loaded with guest MSR values.
1579 *
1580 * @param pVCpu The cross context virtual CPU structure.
1581 *
1582 * @remarks No-long-jump zone!!!
1583 * @remarks The guest MSRs should have been saved back into the guest-CPU
1584 * context by hmR0VmxImportGuestState()!!!
1585 */
1586static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1587{
1588 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1589 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1590
1591 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1592 {
1593 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1594#if HC_ARCH_BITS == 64
1595 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1596 {
1597 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1598 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1599 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1600 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1601 }
1602#endif
1603 }
1604 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1605}
1606
1607
1608/**
1609 * Verifies that our cached values of the VMCS fields are all consistent with
1610 * what's actually present in the VMCS.
1611 *
1612 * @returns VBox status code.
1613 * @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
1614 * @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
1615 * VMCS content. HMCPU error-field is
1616 * updated, see VMX_VCI_XXX.
1617 * @param pVCpu The cross context virtual CPU structure.
1618 */
1619static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1620{
1621 uint32_t u32Val;
1622 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1623 AssertRCReturn(rc, rc);
1624 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32EntryCtls == u32Val,
1625 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1626 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
1627 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1628
1629 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1630 AssertRCReturn(rc, rc);
1631 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ExitCtls == u32Val,
1632 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1633 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
1634 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1635
1636 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1637 AssertRCReturn(rc, rc);
1638 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32PinCtls == u32Val,
1639 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1640 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
1641 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1642
1643 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1644 AssertRCReturn(rc, rc);
1645 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls == u32Val,
1646 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1647 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
1648 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1649
1650 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
1651 {
1652 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1653 AssertRCReturn(rc, rc);
1654 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1655 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1656 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
1657 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1658 }
1659
1660 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
1661 AssertRCReturn(rc, rc);
1662 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32XcptBitmap == u32Val,
1663 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap, u32Val),
1664 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
1665 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1666
1667 uint64_t u64Val;
1668 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
1669 AssertRCReturn(rc, rc);
1670 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u64TscOffset == u64Val,
1671 ("Cache=%#RX64 VMCS=%#RX64\n", pVCpu->hm.s.vmx.u64TscOffset, u64Val),
1672 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
1673 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1674
1675 return VINF_SUCCESS;
1676}
1677
1678
1679#ifdef VBOX_STRICT
1680/**
1681 * Verifies that our cached host EFER value has not changed
1682 * since we cached it.
1683 *
1684 * @param pVCpu The cross context virtual CPU structure.
1685 */
1686static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1687{
1688 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1689
1690 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
1691 {
1692 uint64_t u64Val;
1693 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1694 AssertRC(rc);
1695
1696 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1697 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1698 }
1699}
1700
1701
1702/**
1703 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1704 * VMCS are correct.
1705 *
1706 * @param pVCpu The cross context virtual CPU structure.
1707 */
1708static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1709{
1710 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1711
1712 /* Verify MSR counts in the VMCS are what we think it should be. */
1713 uint32_t cMsrs;
1714 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1715 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1716
1717 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1718 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1719
1720 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1721 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1722
1723 PCVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1724 PCVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1725 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1726 {
1727 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1728 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1729 pGuestMsr->u32Msr, cMsrs));
1730
1731 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1732 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1733 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1734
1735 /* Verify that the permissions are as expected in the MSR bitmap. */
1736 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1737 {
1738 VMXMSREXITREAD enmRead;
1739 VMXMSREXITWRITE enmWrite;
1740 rc = HMVmxGetMsrPermission(pVCpu->hm.s.vmx.pvMsrBitmap, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1741 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("HMVmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1742 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1743 {
1744 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1745 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1746 }
1747 else
1748 {
1749 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1750 pGuestMsr->u32Msr, cMsrs));
1751 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1752 pGuestMsr->u32Msr, cMsrs));
1753 }
1754 }
1755 }
1756}
1757#endif /* VBOX_STRICT */
1758
1759
1760/**
1761 * Flushes the TLB using EPT.
1762 *
1763 * @returns VBox status code.
1764 * @param pVCpu The cross context virtual CPU structure of the calling
1765 * EMT. Can be NULL depending on @a enmTlbFlush.
1766 * @param enmTlbFlush Type of flush.
1767 *
1768 * @remarks Caller is responsible for making sure this function is called only
1769 * when NestedPaging is supported and providing @a enmTlbFlush that is
1770 * supported by the CPU.
1771 * @remarks Can be called with interrupts disabled.
1772 */
1773static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush)
1774{
1775 uint64_t au64Descriptor[2];
1776 if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
1777 au64Descriptor[0] = 0;
1778 else
1779 {
1780 Assert(pVCpu);
1781 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1782 }
1783 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1784
1785 int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
1786 AssertMsg(rc == VINF_SUCCESS,
1787 ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0, rc));
1788
1789 if ( RT_SUCCESS(rc)
1790 && pVCpu)
1791 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1792}
1793
1794
1795/**
1796 * Flushes the TLB using VPID.
1797 *
1798 * @returns VBox status code.
1799 * @param pVCpu The cross context virtual CPU structure of the calling
1800 * EMT. Can be NULL depending on @a enmTlbFlush.
1801 * @param enmTlbFlush Type of flush.
1802 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1803 * on @a enmTlbFlush).
1804 *
1805 * @remarks Can be called with interrupts disabled.
1806 */
1807static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
1808{
1809 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
1810
1811 uint64_t au64Descriptor[2];
1812 if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
1813 {
1814 au64Descriptor[0] = 0;
1815 au64Descriptor[1] = 0;
1816 }
1817 else
1818 {
1819 AssertPtr(pVCpu);
1820 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1821 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1822 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1823 au64Descriptor[1] = GCPtr;
1824 }
1825
1826 int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
1827 AssertMsg(rc == VINF_SUCCESS,
1828 ("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1829
1830 if ( RT_SUCCESS(rc)
1831 && pVCpu)
1832 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1833 NOREF(rc);
1834}
1835
1836
1837/**
1838 * Invalidates a guest page by guest virtual address. Only relevant for
1839 * EPT/VPID, otherwise there is nothing really to invalidate.
1840 *
1841 * @returns VBox status code.
1842 * @param pVCpu The cross context virtual CPU structure.
1843 * @param GCVirt Guest virtual address of the page to invalidate.
1844 */
1845VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
1846{
1847 AssertPtr(pVCpu);
1848 LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
1849
1850 bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1851 if (!fFlushPending)
1852 {
1853 /*
1854 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1855 * the EPT case. See @bugref{6043} and @bugref{6177}.
1856 *
1857 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1858 * as this function maybe called in a loop with individual addresses.
1859 */
1860 PVM pVM = pVCpu->CTX_SUFF(pVM);
1861 if (pVM->hm.s.vmx.fVpid)
1862 {
1863 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1864
1865#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1866 /*
1867 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1868 * where executing INVVPID outside 64-bit mode does not flush translations of
1869 * 64-bit linear addresses, see @bugref{6208#c72}.
1870 */
1871 if (RT_HI_U32(GCVirt))
1872 fVpidFlush = false;
1873#endif
1874
1875 if (fVpidFlush)
1876 {
1877 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
1878 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1879 }
1880 else
1881 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1882 }
1883 else if (pVM->hm.s.fNestedPaging)
1884 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1885 }
1886
1887 return VINF_SUCCESS;
1888}
1889
1890
1891/**
1892 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1893 * case where neither EPT nor VPID is supported by the CPU.
1894 *
1895 * @param pVCpu The cross context virtual CPU structure.
1896 * @param pCpu Pointer to the global HM struct.
1897 *
1898 * @remarks Called with interrupts disabled.
1899 */
1900static void hmR0VmxFlushTaggedTlbNone(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1901{
1902 AssertPtr(pVCpu);
1903 AssertPtr(pCpu);
1904
1905 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1906
1907 Assert(pCpu->idCpu != NIL_RTCPUID);
1908 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1909 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1910 pVCpu->hm.s.fForceTLBFlush = false;
1911 return;
1912}
1913
1914
1915/**
1916 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1917 *
1918 * @param pVCpu The cross context virtual CPU structure.
1919 * @param pCpu Pointer to the global HM CPU struct.
1920 *
1921 * @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
1922 * nomenclature. The reason is, to avoid confusion in compare statements
1923 * since the host-CPU copies are named "ASID".
1924 *
1925 * @remarks Called with interrupts disabled.
1926 */
1927static void hmR0VmxFlushTaggedTlbBoth(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1928{
1929#ifdef VBOX_WITH_STATISTICS
1930 bool fTlbFlushed = false;
1931# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1932# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1933 if (!fTlbFlushed) \
1934 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1935 } while (0)
1936#else
1937# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1938# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1939#endif
1940
1941 AssertPtr(pCpu);
1942 AssertPtr(pVCpu);
1943 Assert(pCpu->idCpu != NIL_RTCPUID);
1944
1945 PVM pVM = pVCpu->CTX_SUFF(pVM);
1946 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1947 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1948 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1949
1950 /*
1951 * Force a TLB flush for the first world-switch if the current CPU differs from the one we
1952 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1953 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1954 * cannot reuse the current ASID anymore.
1955 */
1956 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1957 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1958 {
1959 ++pCpu->uCurrentAsid;
1960 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1961 {
1962 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1963 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1964 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1965 }
1966
1967 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1968 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1969 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1970
1971 /*
1972 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1973 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1974 */
1975 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1976 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1977 HMVMX_SET_TAGGED_TLB_FLUSHED();
1978 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1979 }
1980 else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
1981 {
1982 /*
1983 * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
1984 * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
1985 * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
1986 * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
1987 * mappings, see @bugref{6568}.
1988 *
1989 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
1990 */
1991 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1992 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1993 HMVMX_SET_TAGGED_TLB_FLUSHED();
1994 }
1995
1996 pVCpu->hm.s.fForceTLBFlush = false;
1997 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1998
1999 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2000 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2001 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2002 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2003 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2004 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2005 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2006 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2007 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2008
2009 /* Update VMCS with the VPID. */
2010 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2011 AssertRC(rc);
2012
2013#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2014}
2015
2016
2017/**
2018 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2019 *
2020 * @returns VBox status code.
2021 * @param pVCpu The cross context virtual CPU structure.
2022 * @param pCpu Pointer to the global HM CPU struct.
2023 *
2024 * @remarks Called with interrupts disabled.
2025 */
2026static void hmR0VmxFlushTaggedTlbEpt(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2027{
2028 AssertPtr(pVCpu);
2029 AssertPtr(pCpu);
2030 Assert(pCpu->idCpu != NIL_RTCPUID);
2031 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2032 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2033
2034 /*
2035 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2036 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2037 */
2038 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2039 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2040 {
2041 pVCpu->hm.s.fForceTLBFlush = true;
2042 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2043 }
2044
2045 /* Check for explicit TLB flushes. */
2046 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2047 {
2048 pVCpu->hm.s.fForceTLBFlush = true;
2049 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2050 }
2051
2052 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2053 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2054
2055 if (pVCpu->hm.s.fForceTLBFlush)
2056 {
2057 hmR0VmxFlushEpt(pVCpu, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2058 pVCpu->hm.s.fForceTLBFlush = false;
2059 }
2060}
2061
2062
2063/**
2064 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2065 *
2066 * @returns VBox status code.
2067 * @param pVCpu The cross context virtual CPU structure.
2068 * @param pCpu Pointer to the global HM CPU struct.
2069 *
2070 * @remarks Called with interrupts disabled.
2071 */
2072static void hmR0VmxFlushTaggedTlbVpid(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2073{
2074 AssertPtr(pVCpu);
2075 AssertPtr(pCpu);
2076 Assert(pCpu->idCpu != NIL_RTCPUID);
2077 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2078 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2079
2080 /*
2081 * Force a TLB flush for the first world switch if the current CPU differs from the one we
2082 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2083 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2084 * cannot reuse the current ASID anymore.
2085 */
2086 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2087 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2088 {
2089 pVCpu->hm.s.fForceTLBFlush = true;
2090 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2091 }
2092
2093 /* Check for explicit TLB flushes. */
2094 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2095 {
2096 /*
2097 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2098 * hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2099 * fExplicitFlush = true here and change the pCpu->fFlushAsidBeforeUse check below to
2100 * include fExplicitFlush's too) - an obscure corner case.
2101 */
2102 pVCpu->hm.s.fForceTLBFlush = true;
2103 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2104 }
2105
2106 PVM pVM = pVCpu->CTX_SUFF(pVM);
2107 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2108 if (pVCpu->hm.s.fForceTLBFlush)
2109 {
2110 ++pCpu->uCurrentAsid;
2111 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2112 {
2113 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2114 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2115 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2116 }
2117
2118 pVCpu->hm.s.fForceTLBFlush = false;
2119 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2120 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2121 if (pCpu->fFlushAsidBeforeUse)
2122 {
2123 if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2124 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2125 else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2126 {
2127 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2128 pCpu->fFlushAsidBeforeUse = false;
2129 }
2130 else
2131 {
2132 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2133 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2134 }
2135 }
2136 }
2137
2138 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2139 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2140 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2141 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2142 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2143 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2144 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2145
2146 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2147 AssertRC(rc);
2148}
2149
2150
2151/**
2152 * Flushes the guest TLB entry based on CPU capabilities.
2153 *
2154 * @param pVCpu The cross context virtual CPU structure.
2155 * @param pCpu Pointer to the global HM CPU struct.
2156 */
2157DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2158{
2159#ifdef HMVMX_ALWAYS_FLUSH_TLB
2160 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2161#endif
2162 PVM pVM = pVCpu->CTX_SUFF(pVM);
2163 switch (pVM->hm.s.vmx.enmTlbFlushType)
2164 {
2165 case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVCpu, pCpu); break;
2166 case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pVCpu, pCpu); break;
2167 case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pVCpu, pCpu); break;
2168 case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pVCpu, pCpu); break;
2169 default:
2170 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2171 break;
2172 }
2173 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2174}
2175
2176
2177/**
2178 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2179 * TLB entries from the host TLB before VM-entry.
2180 *
2181 * @returns VBox status code.
2182 * @param pVM The cross context VM structure.
2183 */
2184static int hmR0VmxSetupTaggedTlb(PVM pVM)
2185{
2186 /*
2187 * Determine optimal flush type for Nested Paging.
2188 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2189 * guest execution (see hmR3InitFinalizeR0()).
2190 */
2191 if (pVM->hm.s.fNestedPaging)
2192 {
2193 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2194 {
2195 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2196 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2197 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2198 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2199 else
2200 {
2201 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2202 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2203 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2204 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2205 }
2206
2207 /* Make sure the write-back cacheable memory type for EPT is supported. */
2208 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2209 {
2210 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2211 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2212 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2213 }
2214
2215 /* EPT requires a page-walk length of 4. */
2216 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2217 {
2218 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2219 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2220 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2221 }
2222 }
2223 else
2224 {
2225 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2226 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2227 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2228 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2229 }
2230 }
2231
2232 /*
2233 * Determine optimal flush type for VPID.
2234 */
2235 if (pVM->hm.s.vmx.fVpid)
2236 {
2237 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2238 {
2239 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2240 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2241 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2242 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2243 else
2244 {
2245 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2246 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2247 LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2248 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2249 LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2250 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2251 pVM->hm.s.vmx.fVpid = false;
2252 }
2253 }
2254 else
2255 {
2256 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2257 Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2258 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2259 pVM->hm.s.vmx.fVpid = false;
2260 }
2261 }
2262
2263 /*
2264 * Setup the handler for flushing tagged-TLBs.
2265 */
2266 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2267 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2268 else if (pVM->hm.s.fNestedPaging)
2269 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2270 else if (pVM->hm.s.vmx.fVpid)
2271 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2272 else
2273 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2274 return VINF_SUCCESS;
2275}
2276
2277
2278/**
2279 * Sets up pin-based VM-execution controls in the VMCS.
2280 *
2281 * @returns VBox status code.
2282 * @param pVCpu The cross context virtual CPU structure.
2283 *
2284 * @remarks We don't really care about optimizing vmwrites here as it's done only
2285 * once per VM and hence we don't care about VMCS-field cache comparisons.
2286 */
2287static int hmR0VmxSetupPinCtls(PVMCPU pVCpu)
2288{
2289 PVM pVM = pVCpu->CTX_SUFF(pVM);
2290 uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.disallowed0; /* Bits set here must always be set. */
2291 uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2292
2293 fVal |= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2294 | VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2295
2296 if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2297 fVal |= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2298
2299 /* Enable the VMX preemption timer. */
2300 if (pVM->hm.s.vmx.fUsePreemptTimer)
2301 {
2302 Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2303 fVal |= VMX_PIN_CTLS_PREEMPT_TIMER;
2304 }
2305
2306#if 0
2307 /* Enable posted-interrupt processing. */
2308 if (pVM->hm.s.fPostedIntrs)
2309 {
2310 Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2311 Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2312 fVal |= VMX_PIN_CTL_POSTED_INT;
2313 }
2314#endif
2315
2316 if ((fVal & fZap) != fVal)
2317 {
2318 LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2319 pVM->hm.s.vmx.Msrs.PinCtls.n.disallowed0, fVal, fZap));
2320 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2321 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2322 }
2323
2324 /* Commit it to the VMCS and update our cache. */
2325 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2326 AssertRCReturn(rc, rc);
2327 pVCpu->hm.s.vmx.u32PinCtls = fVal;
2328
2329 return VINF_SUCCESS;
2330}
2331
2332
2333/**
2334 * Sets up secondary processor-based VM-execution controls in the VMCS.
2335 *
2336 * @returns VBox status code.
2337 * @param pVCpu The cross context virtual CPU structure.
2338 *
2339 * @remarks We don't really care about optimizing vmwrites here as it's done only
2340 * once per VM and hence we don't care about VMCS-field cache comparisons.
2341 */
2342static int hmR0VmxSetupProcCtls2(PVMCPU pVCpu)
2343{
2344 PVM pVM = pVCpu->CTX_SUFF(pVM);
2345 uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2346 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2347
2348 /* WBINVD causes a VM-exit. */
2349 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2350 fVal |= VMX_PROC_CTLS2_WBINVD_EXIT;
2351
2352 /* Enable EPT (aka nested-paging). */
2353 if (pVM->hm.s.fNestedPaging)
2354 fVal |= VMX_PROC_CTLS2_EPT;
2355
2356 /*
2357 * Enable the INVPCID instruction if supported by the hardware and we expose
2358 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2359 */
2360 if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
2361 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2362 fVal |= VMX_PROC_CTLS2_INVPCID;
2363
2364 /* Enable VPID. */
2365 if (pVM->hm.s.vmx.fVpid)
2366 fVal |= VMX_PROC_CTLS2_VPID;
2367
2368 /* Enable Unrestricted guest execution. */
2369 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2370 fVal |= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
2371
2372#if 0
2373 if (pVM->hm.s.fVirtApicRegs)
2374 {
2375 /* Enable APIC-register virtualization. */
2376 Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
2377 fVal |= VMX_PROC_CTLS2_APIC_REG_VIRT;
2378
2379 /* Enable virtual-interrupt delivery. */
2380 Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
2381 fVal |= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
2382 }
2383#endif
2384
2385 /* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
2386 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2387 * done dynamically. */
2388 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
2389 {
2390 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2391 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2392 fVal |= VMX_PROC_CTLS2_VIRT_APIC_ACCESS; /* Virtualize APIC accesses. */
2393 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2394 AssertRCReturn(rc, rc);
2395 }
2396
2397 /* Enable RDTSCP. */
2398 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
2399 fVal |= VMX_PROC_CTLS2_RDTSCP;
2400
2401 /* Enable Pause-Loop exiting. */
2402 if ( pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT
2403 && pVM->hm.s.vmx.cPleGapTicks
2404 && pVM->hm.s.vmx.cPleWindowTicks)
2405 {
2406 fVal |= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
2407
2408 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2409 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2410 AssertRCReturn(rc, rc);
2411 }
2412
2413 if ((fVal & fZap) != fVal)
2414 {
2415 LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2416 pVM->hm.s.vmx.Msrs.ProcCtls2.n.disallowed0, fVal, fZap));
2417 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2418 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2419 }
2420
2421 /* Commit it to the VMCS and update our cache. */
2422 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2423 AssertRCReturn(rc, rc);
2424 pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2425
2426 return VINF_SUCCESS;
2427}
2428
2429
2430/**
2431 * Sets up processor-based VM-execution controls in the VMCS.
2432 *
2433 * @returns VBox status code.
2434 * @param pVCpu The cross context virtual CPU structure.
2435 *
2436 * @remarks We don't really care about optimizing vmwrites here as it's done only
2437 * once per VM and hence we don't care about VMCS-field cache comparisons.
2438 */
2439static int hmR0VmxSetupProcCtls(PVMCPU pVCpu)
2440{
2441 PVM pVM = pVCpu->CTX_SUFF(pVM);
2442 uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2443 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2444
2445 fVal |= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
2446 | VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2447 | VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2448 | VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2449 | VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2450 | VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2451 | VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2452
2453 /* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2454 if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
2455 || (pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
2456 {
2457 LogRelFunc(("Unsupported VMX_PROC_CTLS_MOV_DR_EXIT combo!"));
2458 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2459 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2460 }
2461
2462 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2463 if (!pVM->hm.s.fNestedPaging)
2464 {
2465 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2466 fVal |= VMX_PROC_CTLS_INVLPG_EXIT
2467 | VMX_PROC_CTLS_CR3_LOAD_EXIT
2468 | VMX_PROC_CTLS_CR3_STORE_EXIT;
2469 }
2470
2471 /* Use TPR shadowing if supported by the CPU. */
2472 if ( PDMHasApic(pVM)
2473 && pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
2474 {
2475 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2476 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2477 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2478 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2479 AssertRCReturn(rc, rc);
2480
2481 fVal |= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2482 /* CR8 writes cause a VM-exit based on TPR threshold. */
2483 Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
2484 Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
2485 }
2486 else
2487 {
2488 /*
2489 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2490 * Set this control only for 64-bit guests.
2491 */
2492 if (pVM->hm.s.fAllow64BitGuests)
2493 {
2494 fVal |= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2495 | VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2496 }
2497 }
2498
2499 /* Use MSR-bitmaps if supported by the CPU. */
2500 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2501 {
2502 fVal |= VMX_PROC_CTLS_USE_MSR_BITMAPS;
2503
2504 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2505 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2506 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2507 AssertRCReturn(rc, rc);
2508
2509 /*
2510 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2511 * automatically using dedicated fields in the VMCS.
2512 */
2513 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2514 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2515 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2516 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2517 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2518#if HC_ARCH_BITS == 64
2519 /*
2520 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2521 */
2522 if (pVM->hm.s.fAllow64BitGuests)
2523 {
2524 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2525 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2526 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2527 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2528 }
2529#endif
2530 /*
2531 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2532 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2533 */
2534 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2535 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2536
2537 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2538 }
2539
2540 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2541 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2542 fVal |= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
2543
2544 if ((fVal & fZap) != fVal)
2545 {
2546 LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2547 pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0, fVal, fZap));
2548 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2549 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2550 }
2551
2552 /* Commit it to the VMCS and update our cache. */
2553 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2554 AssertRCReturn(rc, rc);
2555 pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2556
2557 /* Set up secondary processor-based VM-execution controls if the CPU supports it. */
2558 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2559 return hmR0VmxSetupProcCtls2(pVCpu);
2560
2561 /* Sanity check, should not really happen. */
2562 if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2563 {
2564 LogRelFunc(("Unrestricted Guest enabled when secondary processor-based VM-execution controls not available\n"));
2565 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2566 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2567 }
2568
2569 /* Old CPUs without secondary processor-based VM-execution controls would end up here. */
2570 return VINF_SUCCESS;
2571}
2572
2573
2574/**
2575 * Sets up miscellaneous (everything other than Pin & Processor-based
2576 * VM-execution) control fields in the VMCS.
2577 *
2578 * @returns VBox status code.
2579 * @param pVCpu The cross context virtual CPU structure.
2580 */
2581static int hmR0VmxSetupMiscCtls(PVMCPU pVCpu)
2582{
2583 AssertPtr(pVCpu);
2584
2585 int rc = VERR_GENERAL_FAILURE;
2586
2587 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2588#if 0
2589 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2590 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2591 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2592
2593 /*
2594 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2595 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2596 * We thus use the exception bitmap to control it rather than use both.
2597 */
2598 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2599 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2600
2601 /* All IO & IOIO instructions cause VM-exits. */
2602 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2603 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2604
2605 /* Initialize the MSR-bitmap area. */
2606 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2607 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2608 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2609 AssertRCReturn(rc, rc);
2610#endif
2611
2612 /* Setup MSR auto-load/store area. */
2613 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2614 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2615 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2616 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2617 AssertRCReturn(rc, rc);
2618
2619 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2620 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2621 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2622 AssertRCReturn(rc, rc);
2623
2624 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2625 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2626 AssertRCReturn(rc, rc);
2627
2628 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2629#if 0
2630 /* Setup debug controls */
2631 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2632 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, 0);
2633 AssertRCReturn(rc, rc);
2634#endif
2635
2636 return rc;
2637}
2638
2639
2640/**
2641 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2642 *
2643 * We shall setup those exception intercepts that don't change during the
2644 * lifetime of the VM here. The rest are done dynamically while loading the
2645 * guest state.
2646 *
2647 * @returns VBox status code.
2648 * @param pVCpu The cross context virtual CPU structure.
2649 */
2650static int hmR0VmxInitXcptBitmap(PVMCPU pVCpu)
2651{
2652 AssertPtr(pVCpu);
2653
2654 uint32_t uXcptBitmap;
2655
2656 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2657 uXcptBitmap = RT_BIT_32(X86_XCPT_AC);
2658
2659 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2660 and writes, and because recursive #DBs can cause the CPU hang, we must always
2661 intercept #DB. */
2662 uXcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2663
2664 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2665 if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
2666 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
2667
2668 /* Commit it to the VMCS. */
2669 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2670 AssertRCReturn(rc, rc);
2671
2672 /* Update our cache of the exception bitmap. */
2673 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
2674 return VINF_SUCCESS;
2675}
2676
2677
2678/**
2679 * Does per-VM VT-x initialization.
2680 *
2681 * @returns VBox status code.
2682 * @param pVM The cross context VM structure.
2683 */
2684VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2685{
2686 LogFlowFunc(("pVM=%p\n", pVM));
2687
2688 int rc = hmR0VmxStructsAlloc(pVM);
2689 if (RT_FAILURE(rc))
2690 {
2691 LogRelFunc(("hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2692 return rc;
2693 }
2694
2695 return VINF_SUCCESS;
2696}
2697
2698
2699/**
2700 * Does per-VM VT-x termination.
2701 *
2702 * @returns VBox status code.
2703 * @param pVM The cross context VM structure.
2704 */
2705VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2706{
2707 LogFlowFunc(("pVM=%p\n", pVM));
2708
2709#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2710 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2711 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2712#endif
2713 hmR0VmxStructsFree(pVM);
2714 return VINF_SUCCESS;
2715}
2716
2717
2718/**
2719 * Sets up the VM for execution under VT-x.
2720 * This function is only called once per-VM during initialization.
2721 *
2722 * @returns VBox status code.
2723 * @param pVM The cross context VM structure.
2724 */
2725VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2726{
2727 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2728 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2729
2730 LogFlowFunc(("pVM=%p\n", pVM));
2731
2732 /*
2733 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be
2734 * allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2735 * pRealModeTSS, see hmR3InitFinalizeR0Intel().
2736 */
2737 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2738 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2739 || !pVM->hm.s.vmx.pRealModeTSS))
2740 {
2741 LogRelFunc(("Invalid real-on-v86 state.\n"));
2742 return VERR_INTERNAL_ERROR;
2743 }
2744
2745 /* Initialize these always, see hmR3InitFinalizeR0().*/
2746 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
2747 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
2748
2749 /* Setup the tagged-TLB flush handlers. */
2750 int rc = hmR0VmxSetupTaggedTlb(pVM);
2751 if (RT_FAILURE(rc))
2752 {
2753 LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2754 return rc;
2755 }
2756
2757 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2758 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2759#if HC_ARCH_BITS == 64
2760 if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
2761 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2762 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
2763 {
2764 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2765 }
2766#endif
2767
2768 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2769 RTCCUINTREG const uHostCR4 = ASMGetCR4();
2770 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2771 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2772
2773 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2774 {
2775 PVMCPU pVCpu = &pVM->aCpus[i];
2776 AssertPtr(pVCpu);
2777 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2778
2779 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2780 Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2781
2782 /* Set revision dword at the beginning of the VMCS structure. */
2783 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
2784
2785 /* Set the VMCS launch state to "clear", see Intel spec. 31.6 "Preparation and launch a virtual machine". */
2786 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2787 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc\n", rc),
2788 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2789
2790 /* Load this VMCS as the current VMCS. */
2791 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2792 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc\n", rc),
2793 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2794
2795 rc = hmR0VmxSetupPinCtls(pVCpu);
2796 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc\n", rc),
2797 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2798
2799 rc = hmR0VmxSetupProcCtls(pVCpu);
2800 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc\n", rc),
2801 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2802
2803 rc = hmR0VmxSetupMiscCtls(pVCpu);
2804 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc\n", rc),
2805 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2806
2807 rc = hmR0VmxInitXcptBitmap(pVCpu);
2808 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc\n", rc),
2809 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2810
2811#if HC_ARCH_BITS == 32
2812 rc = hmR0VmxInitVmcsReadCache(pVCpu);
2813 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc\n", rc),
2814 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2815#endif
2816
2817 /* Sync any CPU internal VMCS data back into our VMCS in memory. */
2818 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2819 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc\n", rc),
2820 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2821
2822 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2823
2824 hmR0VmxUpdateErrorRecord(pVCpu, rc);
2825 }
2826
2827 return VINF_SUCCESS;
2828}
2829
2830
2831/**
2832 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2833 * the VMCS.
2834 *
2835 * @returns VBox status code.
2836 */
2837static int hmR0VmxExportHostControlRegs(void)
2838{
2839 RTCCUINTREG uReg = ASMGetCR0();
2840 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2841 AssertRCReturn(rc, rc);
2842
2843 uReg = ASMGetCR3();
2844 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2845 AssertRCReturn(rc, rc);
2846
2847 uReg = ASMGetCR4();
2848 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2849 AssertRCReturn(rc, rc);
2850 return rc;
2851}
2852
2853
2854/**
2855 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2856 * the host-state area in the VMCS.
2857 *
2858 * @returns VBox status code.
2859 * @param pVCpu The cross context virtual CPU structure.
2860 */
2861static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2862{
2863#if HC_ARCH_BITS == 64
2864/**
2865 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2866 * requirements. See hmR0VmxExportHostSegmentRegs().
2867 */
2868# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2869 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2870 { \
2871 bool fValidSelector = true; \
2872 if ((selValue) & X86_SEL_LDT) \
2873 { \
2874 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2875 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2876 } \
2877 if (fValidSelector) \
2878 { \
2879 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2880 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2881 } \
2882 (selValue) = 0; \
2883 }
2884
2885 /*
2886 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2887 * should -not- save the messed up state without restoring the original host-state,
2888 * see @bugref{7240}.
2889 *
2890 * This apparently can happen (most likely the FPU changes), deal with it rather than
2891 * asserting. Was observed booting Solaris 10u10 32-bit guest.
2892 */
2893 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2894 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2895 {
2896 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2897 pVCpu->idCpu));
2898 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2899 }
2900 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2901#else
2902 RT_NOREF(pVCpu);
2903#endif
2904
2905 /*
2906 * Host DS, ES, FS and GS segment registers.
2907 */
2908#if HC_ARCH_BITS == 64
2909 RTSEL uSelDS = ASMGetDS();
2910 RTSEL uSelES = ASMGetES();
2911 RTSEL uSelFS = ASMGetFS();
2912 RTSEL uSelGS = ASMGetGS();
2913#else
2914 RTSEL uSelDS = 0;
2915 RTSEL uSelES = 0;
2916 RTSEL uSelFS = 0;
2917 RTSEL uSelGS = 0;
2918#endif
2919
2920 /*
2921 * Host CS and SS segment registers.
2922 */
2923 RTSEL uSelCS = ASMGetCS();
2924 RTSEL uSelSS = ASMGetSS();
2925
2926 /*
2927 * Host TR segment register.
2928 */
2929 RTSEL uSelTR = ASMGetTR();
2930
2931#if HC_ARCH_BITS == 64
2932 /*
2933 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2934 * gain VM-entry and restore them before we get preempted.
2935 *
2936 * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2937 */
2938 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2939 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2940 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2941 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2942# undef VMXLOCAL_ADJUST_HOST_SEG
2943#endif
2944
2945 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2946 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2947 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2948 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2949 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2950 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2951 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2952 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2953 Assert(uSelCS);
2954 Assert(uSelTR);
2955
2956 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2957#if 0
2958 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE))
2959 Assert(uSelSS != 0);
2960#endif
2961
2962 /* Write these host selector fields into the host-state area in the VMCS. */
2963 int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2964 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2965#if HC_ARCH_BITS == 64
2966 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2967 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2968 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2969 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2970#else
2971 NOREF(uSelDS);
2972 NOREF(uSelES);
2973 NOREF(uSelFS);
2974 NOREF(uSelGS);
2975#endif
2976 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2977 AssertRCReturn(rc, rc);
2978
2979 /*
2980 * Host GDTR and IDTR.
2981 */
2982 RTGDTR Gdtr;
2983 RTIDTR Idtr;
2984 RT_ZERO(Gdtr);
2985 RT_ZERO(Idtr);
2986 ASMGetGDTR(&Gdtr);
2987 ASMGetIDTR(&Idtr);
2988 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2989 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2990 AssertRCReturn(rc, rc);
2991
2992#if HC_ARCH_BITS == 64
2993 /*
2994 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
2995 * them to the maximum limit (0xffff) on every VM-exit.
2996 */
2997 if (Gdtr.cbGdt != 0xffff)
2998 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2999
3000 /*
3001 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
3002 * Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
3003 * as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
3004 * However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
3005 * uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
3006 * alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
3007 * at 0xffff on hosts where we are sure it won't cause trouble.
3008 */
3009# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3010 if (Idtr.cbIdt < 0x0fff)
3011# else
3012 if (Idtr.cbIdt != 0xffff)
3013# endif
3014 {
3015 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3016 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3017 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3018 }
3019#endif
3020
3021 /*
3022 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3023 * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3024 * RPL should be too in most cases.
3025 */
3026 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3027 ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3028
3029 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3030#if HC_ARCH_BITS == 64
3031 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3032
3033 /*
3034 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3035 * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3036 * restoration if the host has something else. Task switching is not supported in 64-bit
3037 * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3038 * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3039 *
3040 * [1] See Intel spec. 3.5 "System Descriptor Types".
3041 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3042 */
3043 PVM pVM = pVCpu->CTX_SUFF(pVM);
3044 Assert(pDesc->System.u4Type == 11);
3045 if ( pDesc->System.u16LimitLow != 0x67
3046 || pDesc->System.u4LimitHigh)
3047 {
3048 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3049 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3050 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3051 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3052 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3053 }
3054
3055 /*
3056 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3057 */
3058 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3059 {
3060 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3061 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3062 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3063 {
3064 /* The GDT is read-only but the writable GDT is available. */
3065 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3066 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3067 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3068 AssertRCReturn(rc, rc);
3069 }
3070 }
3071#else
3072 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3073#endif
3074 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3075 AssertRCReturn(rc, rc);
3076
3077 /*
3078 * Host FS base and GS base.
3079 */
3080#if HC_ARCH_BITS == 64
3081 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3082 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3083 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3084 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3085 AssertRCReturn(rc, rc);
3086
3087 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3088 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3089 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3090 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3091 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3092#endif
3093 return VINF_SUCCESS;
3094}
3095
3096
3097/**
3098 * Exports certain host MSRs in the VM-exit MSR-load area and some in the
3099 * host-state area of the VMCS.
3100 *
3101 * Theses MSRs will be automatically restored on the host after every successful
3102 * VM-exit.
3103 *
3104 * @returns VBox status code.
3105 * @param pVCpu The cross context virtual CPU structure.
3106 *
3107 * @remarks No-long-jump zone!!!
3108 */
3109static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3110{
3111 AssertPtr(pVCpu);
3112 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3113
3114 /*
3115 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3116 * rather than swapping them on every VM-entry.
3117 */
3118 hmR0VmxLazySaveHostMsrs(pVCpu);
3119
3120 /*
3121 * Host Sysenter MSRs.
3122 */
3123 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3124#if HC_ARCH_BITS == 32
3125 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3126 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3127#else
3128 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3129 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3130#endif
3131 AssertRCReturn(rc, rc);
3132
3133 /*
3134 * Host EFER MSR.
3135 *
3136 * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3137 * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3138 */
3139 PVM pVM = pVCpu->CTX_SUFF(pVM);
3140 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3141 {
3142 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3143 AssertRCReturn(rc, rc);
3144 }
3145
3146 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3147
3148 return VINF_SUCCESS;
3149}
3150
3151
3152/**
3153 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3154 *
3155 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3156 * these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3157 * hmR0VMxExportGuestEntryCtls().
3158 *
3159 * @returns true if we need to load guest EFER, false otherwise.
3160 * @param pVCpu The cross context virtual CPU structure.
3161 *
3162 * @remarks Requires EFER, CR4.
3163 * @remarks No-long-jump zone!!!
3164 */
3165static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu)
3166{
3167#ifdef HMVMX_ALWAYS_SWAP_EFER
3168 RT_NOREF(pVCpu);
3169 return true;
3170#else
3171
3172 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3173#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3174 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3175 if (CPUMIsGuestInLongModeEx(pCtx))
3176 return false;
3177#endif
3178
3179 PVM pVM = pVCpu->CTX_SUFF(pVM);
3180 uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3181 uint64_t const u64GuestEfer = pCtx->msrEFER;
3182
3183 /*
3184 * For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3185 * guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3186 */
3187 if ( CPUMIsGuestInLongModeEx(pCtx)
3188 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3189 {
3190 return true;
3191 }
3192
3193 /*
3194 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3195 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3196 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3197 */
3198 if ( (pCtx->cr4 & X86_CR4_PAE)
3199 && (pCtx->cr0 & X86_CR0_PG)
3200 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3201 {
3202 /* Assert that host is NX capable. */
3203 Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
3204 return true;
3205 }
3206
3207 return false;
3208#endif
3209}
3210
3211
3212/**
3213 * Exports the guest state with appropriate VM-entry controls in the VMCS.
3214 *
3215 * These controls can affect things done on VM-exit; e.g. "load debug controls",
3216 * see Intel spec. 24.8.1 "VM-entry controls".
3217 *
3218 * @returns VBox status code.
3219 * @param pVCpu The cross context virtual CPU structure.
3220 *
3221 * @remarks Requires EFER.
3222 * @remarks No-long-jump zone!!!
3223 */
3224static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu)
3225{
3226 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3227 {
3228 PVM pVM = pVCpu->CTX_SUFF(pVM);
3229 uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
3230 uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3231
3232 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3233 fVal |= VMX_ENTRY_CTLS_LOAD_DEBUG;
3234
3235 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3236 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
3237 {
3238 fVal |= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
3239 Log4Func(("VMX_ENTRY_CTLS_IA32E_MODE_GUEST\n"));
3240 }
3241 else
3242 Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
3243
3244 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3245 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3246 && hmR0VmxShouldSwapEferMsr(pVCpu))
3247 {
3248 fVal |= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
3249 Log4Func(("VMX_ENTRY_CTLS_LOAD_EFER_MSR\n"));
3250 }
3251
3252 /*
3253 * The following should -not- be set (since we're not in SMM mode):
3254 * - VMX_ENTRY_CTLS_ENTRY_TO_SMM
3255 * - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
3256 */
3257
3258 /** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
3259 * VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
3260
3261 if ((fVal & fZap) != fVal)
3262 {
3263 Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3264 pVM->hm.s.vmx.Msrs.EntryCtls.n.disallowed0, fVal, fZap));
3265 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3266 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3267 }
3268
3269 /* Commit it to the VMCS and update our cache. */
3270 if (pVCpu->hm.s.vmx.u32EntryCtls != fVal)
3271 {
3272 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3273 AssertRCReturn(rc, rc);
3274 pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3275 }
3276
3277 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3278 }
3279 return VINF_SUCCESS;
3280}
3281
3282
3283/**
3284 * Exports the guest state with appropriate VM-exit controls in the VMCS.
3285 *
3286 * @returns VBox status code.
3287 * @param pVCpu The cross context virtual CPU structure.
3288 *
3289 * @remarks Requires EFER.
3290 */
3291static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu)
3292{
3293 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3294 {
3295 PVM pVM = pVCpu->CTX_SUFF(pVM);
3296 uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
3297 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3298
3299 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3300 fVal |= VMX_EXIT_CTLS_SAVE_DEBUG;
3301
3302 /*
3303 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3304 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3305 * hmR0VmxExportHostMsrs().
3306 */
3307#if HC_ARCH_BITS == 64
3308 fVal |= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
3309 Log4Func(("VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE\n"));
3310#else
3311 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3312 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3313 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3314 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3315 {
3316 /* The switcher returns to long mode, EFER is managed by the switcher. */
3317 fVal |= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
3318 Log4Func(("VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE\n"));
3319 }
3320 else
3321 Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
3322#endif
3323
3324 /* If the newer VMCS fields for managing EFER exists, use it. */
3325 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3326 && hmR0VmxShouldSwapEferMsr(pVCpu))
3327 {
3328 fVal |= VMX_EXIT_CTLS_SAVE_EFER_MSR
3329 | VMX_EXIT_CTLS_LOAD_EFER_MSR;
3330 Log4Func(("VMX_EXIT_CTLS_SAVE_EFER_MSR and VMX_EXIT_CTLS_LOAD_EFER_MSR\n"));
3331 }
3332
3333 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3334 Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
3335
3336 /** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
3337 * VMX_EXIT_CTLS_SAVE_PAT_MSR,
3338 * VMX_EXIT_CTLS_LOAD_PAT_MSR. */
3339
3340 /* Enable saving of the VMX preemption timer value on VM-exit. */
3341 if ( pVM->hm.s.vmx.fUsePreemptTimer
3342 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
3343 fVal |= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
3344
3345 if ((fVal & fZap) != fVal)
3346 {
3347 LogRelFunc(("Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3348 pVM->hm.s.vmx.Msrs.ExitCtls.n.disallowed0, fVal, fZap));
3349 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3350 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3351 }
3352
3353 /* Commit it to the VMCS and update our cache. */
3354 if (pVCpu->hm.s.vmx.u32ExitCtls != fVal)
3355 {
3356 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3357 AssertRCReturn(rc, rc);
3358 pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3359 }
3360
3361 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3362 }
3363 return VINF_SUCCESS;
3364}
3365
3366
3367/**
3368 * Sets the TPR threshold in the VMCS.
3369 *
3370 * @returns VBox status code.
3371 * @param pVCpu The cross context virtual CPU structure.
3372 * @param u32TprThreshold The TPR threshold (task-priority class only).
3373 */
3374DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3375{
3376 Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
3377 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3378 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3379}
3380
3381
3382/**
3383 * Exports the guest APIC TPR state into the VMCS.
3384 *
3385 * @returns VBox status code.
3386 * @param pVCpu The cross context virtual CPU structure.
3387 *
3388 * @remarks No-long-jump zone!!!
3389 */
3390static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3391{
3392 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3393 {
3394 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
3395
3396 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3397 && APICIsEnabled(pVCpu))
3398 {
3399 /*
3400 * Setup TPR shadowing.
3401 */
3402 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
3403 {
3404 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3405
3406 bool fPendingIntr = false;
3407 uint8_t u8Tpr = 0;
3408 uint8_t u8PendingIntr = 0;
3409 int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3410 AssertRCReturn(rc, rc);
3411
3412 /*
3413 * If there are interrupts pending but masked by the TPR, instruct VT-x to
3414 * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3415 * priority of the pending interrupt so we can deliver the interrupt. If there
3416 * are no interrupts pending, set threshold to 0 to not cause any
3417 * TPR-below-threshold VM-exits.
3418 */
3419 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3420 uint32_t u32TprThreshold = 0;
3421 if (fPendingIntr)
3422 {
3423 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3424 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3425 const uint8_t u8TprPriority = u8Tpr >> 4;
3426 if (u8PendingPriority <= u8TprPriority)
3427 u32TprThreshold = u8PendingPriority;
3428 }
3429
3430 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3431 AssertRCReturn(rc, rc);
3432 }
3433 }
3434 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3435 }
3436 return VINF_SUCCESS;
3437}
3438
3439
3440/**
3441 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3442 *
3443 * @returns Guest's interruptibility-state.
3444 * @param pVCpu The cross context virtual CPU structure.
3445 *
3446 * @remarks No-long-jump zone!!!
3447 */
3448static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu)
3449{
3450 /*
3451 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3452 */
3453 uint32_t fIntrState = 0;
3454 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3455 {
3456 /* If inhibition is active, RIP & RFLAGS should've been accessed
3457 (i.e. read previously from the VMCS or from ring-3). */
3458 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3459#ifdef VBOX_STRICT
3460 uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
3461 AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3462#endif
3463 if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3464 {
3465 if (pCtx->eflags.Bits.u1IF)
3466 fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
3467 else
3468 fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
3469 }
3470 else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3471 {
3472 /*
3473 * We can clear the inhibit force flag as even if we go back to the recompiler
3474 * without executing guest code in VT-x, the flag's condition to be cleared is
3475 * met and thus the cleared state is correct.
3476 */
3477 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3478 }
3479 }
3480
3481 /*
3482 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3483 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3484 * setting this would block host-NMIs and IRET will not clear the blocking.
3485 *
3486 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3487 */
3488 if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS)
3489 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
3490 {
3491 fIntrState |= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
3492 }
3493
3494 return fIntrState;
3495}
3496
3497
3498/**
3499 * Exports the exception intercepts required for guest execution in the VMCS.
3500 *
3501 * @returns VBox status code.
3502 * @param pVCpu The cross context virtual CPU structure.
3503 *
3504 * @remarks No-long-jump zone!!!
3505 */
3506static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3507{
3508 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3509 {
3510 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3511
3512 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3513 if (pVCpu->hm.s.fGIMTrapXcptUD)
3514 uXcptBitmap |= RT_BIT(X86_XCPT_UD);
3515#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3516 else
3517 uXcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3518#endif
3519
3520 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_AC));
3521 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_DB));
3522
3523 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3524 {
3525 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3526 AssertRCReturn(rc, rc);
3527 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3528 }
3529
3530 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3531 Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", uXcptBitmap));
3532 }
3533 return VINF_SUCCESS;
3534}
3535
3536
3537/**
3538 * Exports the guest's RIP into the guest-state area in the VMCS.
3539 *
3540 * @returns VBox status code.
3541 * @param pVCpu The cross context virtual CPU structure.
3542 *
3543 * @remarks No-long-jump zone!!!
3544 */
3545static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
3546{
3547 int rc = VINF_SUCCESS;
3548 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3549 {
3550 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
3551
3552 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
3553 AssertRCReturn(rc, rc);
3554
3555 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3556 Log4Func(("RIP=%#RX64\n", pVCpu->cpum.GstCtx.rip));
3557 }
3558 return rc;
3559}
3560
3561
3562/**
3563 * Exports the guest's RSP into the guest-state area in the VMCS.
3564 *
3565 * @returns VBox status code.
3566 * @param pVCpu The cross context virtual CPU structure.
3567 *
3568 * @remarks No-long-jump zone!!!
3569 */
3570static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
3571{
3572 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3573 {
3574 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
3575
3576 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
3577 AssertRCReturn(rc, rc);
3578
3579 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3580 }
3581 return VINF_SUCCESS;
3582}
3583
3584
3585/**
3586 * Exports the guest's RFLAGS into the guest-state area in the VMCS.
3587 *
3588 * @returns VBox status code.
3589 * @param pVCpu The cross context virtual CPU structure.
3590 *
3591 * @remarks No-long-jump zone!!!
3592 */
3593static int hmR0VmxExportGuestRflags(PVMCPU pVCpu)
3594{
3595 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3596 {
3597 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
3598
3599 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3600 Let us assert it as such and use 32-bit VMWRITE. */
3601 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
3602 X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
3603 Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3604 Assert(!(fEFlags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3605
3606 /*
3607 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3608 * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3609 * can run the real-mode guest code under Virtual 8086 mode.
3610 */
3611 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3612 {
3613 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3614 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3615 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3616 fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3617 fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3618 }
3619
3620 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3621 AssertRCReturn(rc, rc);
3622
3623 /*
3624 * Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
3625 *
3626 * We must avoid setting any automatic debug exceptions delivery when single-stepping
3627 * through the hypervisor debugger using EFLAGS.TF.
3628 */
3629 if ( !pVCpu->hm.s.fSingleInstruction
3630 && fEFlags.Bits.u1TF)
3631 {
3632 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
3633 AssertRCReturn(rc, rc);
3634 }
3635
3636 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3637 Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3638 }
3639 return VINF_SUCCESS;
3640}
3641
3642
3643/**
3644 * Exports the guest CR0 control register into the guest-state area in the VMCS.
3645 *
3646 * The guest FPU state is always pre-loaded hence we don't need to bother about
3647 * sharing FPU related CR0 bits between the guest and host.
3648 *
3649 * @returns VBox status code.
3650 * @param pVCpu The cross context virtual CPU structure.
3651 *
3652 * @remarks No-long-jump zone!!!
3653 */
3654static int hmR0VmxExportGuestCR0(PVMCPU pVCpu)
3655{
3656 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3657 {
3658 PVM pVM = pVCpu->CTX_SUFF(pVM);
3659 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
3660 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.cr0));
3661
3662 uint32_t const u32ShadowCr0 = pVCpu->cpum.GstCtx.cr0;
3663 uint32_t u32GuestCr0 = pVCpu->cpum.GstCtx.cr0;
3664
3665 /*
3666 * Setup VT-x's view of the guest CR0.
3667 * Minimize VM-exits due to CR3 changes when we have NestedPaging.
3668 */
3669 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3670 if (pVM->hm.s.fNestedPaging)
3671 {
3672 if (CPUMIsGuestPagingEnabled(pVCpu))
3673 {
3674 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3675 uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
3676 | VMX_PROC_CTLS_CR3_STORE_EXIT);
3677 }
3678 else
3679 {
3680 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3681 uProcCtls |= VMX_PROC_CTLS_CR3_LOAD_EXIT
3682 | VMX_PROC_CTLS_CR3_STORE_EXIT;
3683 }
3684
3685 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3686 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3687 uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
3688 }
3689 else
3690 {
3691 /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3692 u32GuestCr0 |= X86_CR0_WP;
3693 }
3694
3695 /*
3696 * Guest FPU bits.
3697 *
3698 * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3699 * using CR0.TS.
3700 *
3701 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3702 * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3703 */
3704 u32GuestCr0 |= X86_CR0_NE;
3705
3706 /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3707 bool const fInterceptMF = !(u32ShadowCr0 & X86_CR0_NE);
3708
3709 /*
3710 * Update exception intercepts.
3711 */
3712 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3713 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3714 {
3715 Assert(PDMVmmDevHeapIsEnabled(pVM));
3716 Assert(pVM->hm.s.vmx.pRealModeTSS);
3717 uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3718 }
3719 else
3720 {
3721 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3722 uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3723 if (fInterceptMF)
3724 uXcptBitmap |= RT_BIT(X86_XCPT_MF);
3725 }
3726
3727 /* Additional intercepts for debugging, define these yourself explicitly. */
3728#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3729 uXcptBitmap |= 0
3730 | RT_BIT(X86_XCPT_BP)
3731 | RT_BIT(X86_XCPT_DE)
3732 | RT_BIT(X86_XCPT_NM)
3733 | RT_BIT(X86_XCPT_TS)
3734 | RT_BIT(X86_XCPT_UD)
3735 | RT_BIT(X86_XCPT_NP)
3736 | RT_BIT(X86_XCPT_SS)
3737 | RT_BIT(X86_XCPT_GP)
3738 | RT_BIT(X86_XCPT_PF)
3739 | RT_BIT(X86_XCPT_MF)
3740 ;
3741#elif defined(HMVMX_ALWAYS_TRAP_PF)
3742 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
3743#endif
3744 if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
3745 uXcptBitmap |= RT_BIT(X86_XCPT_GP);
3746 Assert(pVM->hm.s.fNestedPaging || (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
3747
3748 /*
3749 * Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3750 */
3751 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3752 uint32_t fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3753 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3754 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
3755 else
3756 Assert((fSetCr0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3757
3758 u32GuestCr0 |= fSetCr0;
3759 u32GuestCr0 &= fZapCr0;
3760 u32GuestCr0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3761
3762 /*
3763 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3764 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3765 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3766 */
3767 uint32_t u32Cr0Mask = X86_CR0_PE
3768 | X86_CR0_NE
3769 | (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3770 | X86_CR0_PG
3771 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3772 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3773 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3774
3775 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3776 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3777 * and @bugref{6944}. */
3778#if 0
3779 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3780 u32Cr0Mask &= ~X86_CR0_PE;
3781#endif
3782 /*
3783 * Finally, update VMCS fields with the CR0 values and the exception bitmap.
3784 */
3785 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCr0);
3786 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32ShadowCr0);
3787 if (u32Cr0Mask != pVCpu->hm.s.vmx.u32Cr0Mask)
3788 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32Cr0Mask);
3789 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3790 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3791 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3792 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3793 AssertRCReturn(rc, rc);
3794
3795 /* Update our caches. */
3796 pVCpu->hm.s.vmx.u32Cr0Mask = u32Cr0Mask;
3797 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3798 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3799
3800 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3801
3802 Log4Func(("u32Cr0Mask=%#RX32 u32ShadowCr0=%#RX32 u32GuestCr0=%#RX32 (fSetCr0=%#RX32 fZapCr0=%#RX32\n", u32Cr0Mask,
3803 u32ShadowCr0, u32GuestCr0, fSetCr0, fZapCr0));
3804 }
3805
3806 return VINF_SUCCESS;
3807}
3808
3809
3810/**
3811 * Exports the guest control registers (CR3, CR4) into the guest-state area
3812 * in the VMCS.
3813 *
3814 * @returns VBox strict status code.
3815 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3816 * without unrestricted guest access and the VMMDev is not presently
3817 * mapped (e.g. EFI32).
3818 *
3819 * @param pVCpu The cross context virtual CPU structure.
3820 *
3821 * @remarks No-long-jump zone!!!
3822 */
3823static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu)
3824{
3825 int rc = VINF_SUCCESS;
3826 PVM pVM = pVCpu->CTX_SUFF(pVM);
3827
3828 /*
3829 * Guest CR2.
3830 * It's always loaded in the assembler code. Nothing to do here.
3831 */
3832
3833 /*
3834 * Guest CR3.
3835 */
3836 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3837 {
3838 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
3839
3840 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3841 if (pVM->hm.s.fNestedPaging)
3842 {
3843 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3844
3845 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3846 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3847 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3848 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3849
3850 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3851 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3852 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3853
3854 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3855 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3856 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3857 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3858 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3859 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3860 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3861
3862 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3863 AssertRCReturn(rc, rc);
3864
3865 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3866 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3867 || CPUMIsGuestPagingEnabledEx(pCtx))
3868 {
3869 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3870 if (CPUMIsGuestInPAEModeEx(pCtx))
3871 {
3872 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3873 AssertRCReturn(rc, rc);
3874 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3875 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3876 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3877 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3878 AssertRCReturn(rc, rc);
3879 }
3880
3881 /*
3882 * The guest's view of its CR3 is unblemished with Nested Paging when the
3883 * guest is using paging or we have unrestricted guest execution to handle
3884 * the guest when it's not using paging.
3885 */
3886 GCPhysGuestCR3 = pCtx->cr3;
3887 }
3888 else
3889 {
3890 /*
3891 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
3892 * thinks it accesses physical memory directly, we use our identity-mapped
3893 * page table to map guest-linear to guest-physical addresses. EPT takes care
3894 * of translating it to host-physical addresses.
3895 */
3896 RTGCPHYS GCPhys;
3897 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3898
3899 /* We obtain it here every time as the guest could have relocated this PCI region. */
3900 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3901 if (RT_SUCCESS(rc))
3902 { /* likely */ }
3903 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3904 {
3905 Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3906 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3907 }
3908 else
3909 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3910
3911 GCPhysGuestCR3 = GCPhys;
3912 }
3913
3914 Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
3915 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3916 AssertRCReturn(rc, rc);
3917 }
3918 else
3919 {
3920 /* Non-nested paging case, just use the hypervisor's CR3. */
3921 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3922
3923 Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
3924 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3925 AssertRCReturn(rc, rc);
3926 }
3927
3928 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3929 }
3930
3931 /*
3932 * Guest CR4.
3933 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3934 */
3935 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3936 {
3937 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3938 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
3939 Assert(!RT_HI_U32(pCtx->cr4));
3940
3941 uint32_t u32GuestCr4 = pCtx->cr4;
3942 uint32_t const u32ShadowCr4 = pCtx->cr4;
3943
3944 /*
3945 * Setup VT-x's view of the guest CR4.
3946 *
3947 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3948 * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3949 * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3950 *
3951 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3952 */
3953 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3954 {
3955 Assert(pVM->hm.s.vmx.pRealModeTSS);
3956 Assert(PDMVmmDevHeapIsEnabled(pVM));
3957 u32GuestCr4 &= ~X86_CR4_VME;
3958 }
3959
3960 if (pVM->hm.s.fNestedPaging)
3961 {
3962 if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3963 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3964 {
3965 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3966 u32GuestCr4 |= X86_CR4_PSE;
3967 /* Our identity mapping is a 32-bit page directory. */
3968 u32GuestCr4 &= ~X86_CR4_PAE;
3969 }
3970 /* else use guest CR4.*/
3971 }
3972 else
3973 {
3974 /*
3975 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3976 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3977 */
3978 switch (pVCpu->hm.s.enmShadowMode)
3979 {
3980 case PGMMODE_REAL: /* Real-mode. */
3981 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3982 case PGMMODE_32_BIT: /* 32-bit paging. */
3983 {
3984 u32GuestCr4 &= ~X86_CR4_PAE;
3985 break;
3986 }
3987
3988 case PGMMODE_PAE: /* PAE paging. */
3989 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3990 {
3991 u32GuestCr4 |= X86_CR4_PAE;
3992 break;
3993 }
3994
3995 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3996 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3997#ifdef VBOX_ENABLE_64_BITS_GUESTS
3998 break;
3999#endif
4000 default:
4001 AssertFailed();
4002 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4003 }
4004 }
4005
4006 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4007 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4008 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4009 u32GuestCr4 |= fSetCr4;
4010 u32GuestCr4 &= fZapCr4;
4011
4012 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them,
4013 that would cause a VM-exit. */
4014 uint32_t u32Cr4Mask = X86_CR4_VME
4015 | X86_CR4_PAE
4016 | X86_CR4_PGE
4017 | X86_CR4_PSE
4018 | X86_CR4_VMXE;
4019 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4020 u32Cr4Mask |= X86_CR4_OSXSAVE;
4021 if (pVM->cpum.ro.GuestFeatures.fPcid)
4022 u32Cr4Mask |= X86_CR4_PCIDE;
4023
4024 /* Write VT-x's view of the guest CR4, the CR4 modify mask and the read-only CR4 shadow
4025 into the VMCS and update our cache. */
4026 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCr4);
4027 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32ShadowCr4);
4028 if (pVCpu->hm.s.vmx.u32Cr4Mask != u32Cr4Mask)
4029 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32Cr4Mask);
4030 AssertRCReturn(rc, rc);
4031 pVCpu->hm.s.vmx.u32Cr4Mask = u32Cr4Mask;
4032
4033 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4034 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
4035
4036 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4037
4038 Log4Func(("u32GuestCr4=%#RX32 u32ShadowCr4=%#RX32 (fSetCr4=%#RX32 fZapCr4=%#RX32)\n", u32GuestCr4, u32ShadowCr4, fSetCr4,
4039 fZapCr4));
4040 }
4041 return rc;
4042}
4043
4044
4045/**
4046 * Exports the guest debug registers into the guest-state area in the VMCS.
4047 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4048 *
4049 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4050 *
4051 * @returns VBox status code.
4052 * @param pVCpu The cross context virtual CPU structure.
4053 *
4054 * @remarks No-long-jump zone!!!
4055 */
4056static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu)
4057{
4058 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4059
4060#ifdef VBOX_STRICT
4061 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4062 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
4063 {
4064 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4065 Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0);
4066 Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
4067 }
4068#endif
4069
4070 bool fSteppingDB = false;
4071 bool fInterceptMovDRx = false;
4072 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4073 if (pVCpu->hm.s.fSingleInstruction)
4074 {
4075 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4076 PVM pVM = pVCpu->CTX_SUFF(pVM);
4077 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
4078 {
4079 uProcCtls |= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
4080 Assert(fSteppingDB == false);
4081 }
4082 else
4083 {
4084 pVCpu->cpum.GstCtx.eflags.u32 |= X86_EFL_TF;
4085 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
4086 pVCpu->hm.s.fClearTrapFlag = true;
4087 fSteppingDB = true;
4088 }
4089 }
4090
4091 uint32_t u32GuestDr7;
4092 if ( fSteppingDB
4093 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4094 {
4095 /*
4096 * Use the combined guest and host DRx values found in the hypervisor register set
4097 * because the debugger has breakpoints active or someone is single stepping on the
4098 * host side without a monitor trap flag.
4099 *
4100 * Note! DBGF expects a clean DR6 state before executing guest code.
4101 */
4102#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4103 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4104 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4105 {
4106 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4107 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4108 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4109 }
4110 else
4111#endif
4112 if (!CPUMIsHyperDebugStateActive(pVCpu))
4113 {
4114 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4115 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4116 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4117 }
4118
4119 /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4120 u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4121 pVCpu->hm.s.fUsingHyperDR7 = true;
4122 fInterceptMovDRx = true;
4123 }
4124 else
4125 {
4126 /*
4127 * If the guest has enabled debug registers, we need to load them prior to
4128 * executing guest code so they'll trigger at the right time.
4129 */
4130 if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD))
4131 {
4132#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4133 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4134 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4135 {
4136 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4137 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4138 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4139 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4140 }
4141 else
4142#endif
4143 if (!CPUMIsGuestDebugStateActive(pVCpu))
4144 {
4145 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4146 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4147 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4148 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4149 }
4150 Assert(!fInterceptMovDRx);
4151 }
4152 /*
4153 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4154 * must intercept #DB in order to maintain a correct DR6 guest value, and
4155 * because we need to intercept it to prevent nested #DBs from hanging the
4156 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4157 */
4158#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4159 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4160 && !CPUMIsGuestDebugStateActive(pVCpu))
4161#else
4162 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4163#endif
4164 {
4165 fInterceptMovDRx = true;
4166 }
4167
4168 /* Update DR7 with the actual guest value. */
4169 u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
4170 pVCpu->hm.s.fUsingHyperDR7 = false;
4171 }
4172
4173 if (fInterceptMovDRx)
4174 uProcCtls |= VMX_PROC_CTLS_MOV_DR_EXIT;
4175 else
4176 uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
4177
4178 /*
4179 * Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
4180 * monitor-trap flag and update our cache.
4181 */
4182 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4183 {
4184 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4185 AssertRCReturn(rc2, rc2);
4186 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4187 }
4188
4189 /*
4190 * Update guest DR7.
4191 */
4192 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
4193 AssertRCReturn(rc, rc);
4194
4195 /*
4196 * If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
4197 * we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
4198 *
4199 * See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
4200 */
4201 if (fSteppingDB)
4202 {
4203 Assert(pVCpu->hm.s.fSingleInstruction);
4204 Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
4205
4206 uint32_t fIntrState = 0;
4207 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
4208 AssertRCReturn(rc, rc);
4209
4210 if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
4211 {
4212 fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
4213 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
4214 AssertRCReturn(rc, rc);
4215 }
4216 }
4217
4218 return VINF_SUCCESS;
4219}
4220
4221
4222#ifdef VBOX_STRICT
4223/**
4224 * Strict function to validate segment registers.
4225 *
4226 * @param pVCpu The cross context virtual CPU structure.
4227 *
4228 * @remarks Will import guest CR0 on strict builds during validation of
4229 * segments.
4230 */
4231static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu)
4232{
4233 /*
4234 * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4235 *
4236 * The reason we check for attribute value 0 in this function and not just the unusable bit is
4237 * because hmR0VmxExportGuestSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4238 * and doesn't change the guest-context value.
4239 */
4240 PVM pVM = pVCpu->CTX_SUFF(pVM);
4241 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4242 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4243 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4244 && ( !CPUMIsGuestInRealModeEx(pCtx)
4245 && !CPUMIsGuestInV86ModeEx(pCtx)))
4246 {
4247 /* Protected mode checks */
4248 /* CS */
4249 Assert(pCtx->cs.Attr.n.u1Present);
4250 Assert(!(pCtx->cs.Attr.u & 0xf00));
4251 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4252 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4253 || !(pCtx->cs.Attr.n.u1Granularity));
4254 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4255 || (pCtx->cs.Attr.n.u1Granularity));
4256 /* CS cannot be loaded with NULL in protected mode. */
4257 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4258 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4259 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4260 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4261 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4262 else
4263 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4264 /* SS */
4265 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4266 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4267 if ( !(pCtx->cr0 & X86_CR0_PE)
4268 || pCtx->cs.Attr.n.u4Type == 3)
4269 {
4270 Assert(!pCtx->ss.Attr.n.u2Dpl);
4271 }
4272 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4273 {
4274 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4275 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4276 Assert(pCtx->ss.Attr.n.u1Present);
4277 Assert(!(pCtx->ss.Attr.u & 0xf00));
4278 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4279 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4280 || !(pCtx->ss.Attr.n.u1Granularity));
4281 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4282 || (pCtx->ss.Attr.n.u1Granularity));
4283 }
4284 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmentReg(). */
4285 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4286 {
4287 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4288 Assert(pCtx->ds.Attr.n.u1Present);
4289 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4290 Assert(!(pCtx->ds.Attr.u & 0xf00));
4291 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4292 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4293 || !(pCtx->ds.Attr.n.u1Granularity));
4294 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4295 || (pCtx->ds.Attr.n.u1Granularity));
4296 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4297 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4298 }
4299 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4300 {
4301 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4302 Assert(pCtx->es.Attr.n.u1Present);
4303 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4304 Assert(!(pCtx->es.Attr.u & 0xf00));
4305 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4306 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4307 || !(pCtx->es.Attr.n.u1Granularity));
4308 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4309 || (pCtx->es.Attr.n.u1Granularity));
4310 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4311 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4312 }
4313 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4314 {
4315 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4316 Assert(pCtx->fs.Attr.n.u1Present);
4317 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4318 Assert(!(pCtx->fs.Attr.u & 0xf00));
4319 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4320 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4321 || !(pCtx->fs.Attr.n.u1Granularity));
4322 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4323 || (pCtx->fs.Attr.n.u1Granularity));
4324 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4325 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4326 }
4327 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4328 {
4329 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4330 Assert(pCtx->gs.Attr.n.u1Present);
4331 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4332 Assert(!(pCtx->gs.Attr.u & 0xf00));
4333 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4334 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4335 || !(pCtx->gs.Attr.n.u1Granularity));
4336 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4337 || (pCtx->gs.Attr.n.u1Granularity));
4338 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4339 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4340 }
4341 /* 64-bit capable CPUs. */
4342# if HC_ARCH_BITS == 64
4343 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4344 Assert(!pCtx->ss.Attr.u || !RT_HI_U32(pCtx->ss.u64Base));
4345 Assert(!pCtx->ds.Attr.u || !RT_HI_U32(pCtx->ds.u64Base));
4346 Assert(!pCtx->es.Attr.u || !RT_HI_U32(pCtx->es.u64Base));
4347# endif
4348 }
4349 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4350 || ( CPUMIsGuestInRealModeEx(pCtx)
4351 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4352 {
4353 /* Real and v86 mode checks. */
4354 /* hmR0VmxExportGuestSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4355 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4356 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4357 {
4358 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4359 }
4360 else
4361 {
4362 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4363 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4364 }
4365
4366 /* CS */
4367 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4368 Assert(pCtx->cs.u32Limit == 0xffff);
4369 Assert(u32CSAttr == 0xf3);
4370 /* SS */
4371 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4372 Assert(pCtx->ss.u32Limit == 0xffff);
4373 Assert(u32SSAttr == 0xf3);
4374 /* DS */
4375 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4376 Assert(pCtx->ds.u32Limit == 0xffff);
4377 Assert(u32DSAttr == 0xf3);
4378 /* ES */
4379 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4380 Assert(pCtx->es.u32Limit == 0xffff);
4381 Assert(u32ESAttr == 0xf3);
4382 /* FS */
4383 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4384 Assert(pCtx->fs.u32Limit == 0xffff);
4385 Assert(u32FSAttr == 0xf3);
4386 /* GS */
4387 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4388 Assert(pCtx->gs.u32Limit == 0xffff);
4389 Assert(u32GSAttr == 0xf3);
4390 /* 64-bit capable CPUs. */
4391# if HC_ARCH_BITS == 64
4392 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4393 Assert(!u32SSAttr || !RT_HI_U32(pCtx->ss.u64Base));
4394 Assert(!u32DSAttr || !RT_HI_U32(pCtx->ds.u64Base));
4395 Assert(!u32ESAttr || !RT_HI_U32(pCtx->es.u64Base));
4396# endif
4397 }
4398}
4399#endif /* VBOX_STRICT */
4400
4401
4402/**
4403 * Exports a guest segment register into the guest-state area in the VMCS.
4404 *
4405 * @returns VBox status code.
4406 * @param pVCpu The cross context virtual CPU structure.
4407 * @param idxSel Index of the selector in the VMCS.
4408 * @param idxLimit Index of the segment limit in the VMCS.
4409 * @param idxBase Index of the segment base in the VMCS.
4410 * @param idxAccess Index of the access rights of the segment in the VMCS.
4411 * @param pSelReg Pointer to the segment selector.
4412 *
4413 * @remarks No-long-jump zone!!!
4414 */
4415static int hmR0VmxExportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
4416 PCCPUMSELREG pSelReg)
4417{
4418 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4419 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4420 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4421 AssertRCReturn(rc, rc);
4422
4423 uint32_t u32Access = pSelReg->Attr.u;
4424 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4425 {
4426 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4427 u32Access = 0xf3;
4428 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4429 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4430 }
4431 else
4432 {
4433 /*
4434 * The way to differentiate between whether this is really a null selector or was just
4435 * a selector loaded with 0 in real-mode is using the segment attributes. A selector
4436 * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4437 * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4438 * NULL selectors loaded in protected-mode have their attribute as 0.
4439 */
4440 if (!u32Access)
4441 u32Access = X86DESCATTR_UNUSABLE;
4442 }
4443
4444 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4445 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4446 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4447
4448 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4449 AssertRCReturn(rc, rc);
4450 return rc;
4451}
4452
4453
4454/**
4455 * Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4456 * into the guest-state area in the VMCS.
4457 *
4458 * @returns VBox status code.
4459 * @param pVCpu The cross context virtual CPU structure.
4460 *
4461 * @remarks Will import guest CR0 on strict builds during validation of
4462 * segments.
4463 * @remarks No-long-jump zone!!!
4464 */
4465static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu)
4466{
4467 int rc = VERR_INTERNAL_ERROR_5;
4468 PVM pVM = pVCpu->CTX_SUFF(pVM);
4469 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4470
4471 /*
4472 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4473 */
4474 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4475 {
4476#ifdef VBOX_WITH_REM
4477 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4478 {
4479 Assert(pVM->hm.s.vmx.pRealModeTSS);
4480 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4481 if ( pVCpu->hm.s.vmx.fWasInRealMode
4482 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4483 {
4484 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4485 in real-mode (e.g. OpenBSD 4.0) */
4486 REMFlushTBs(pVM);
4487 Log4Func(("Switch to protected mode detected!\n"));
4488 pVCpu->hm.s.vmx.fWasInRealMode = false;
4489 }
4490 }
4491#endif
4492 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
4493 {
4494 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
4495 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4496 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pCtx->cs.Attr.u;
4497 rc = HMVMX_EXPORT_SREG(CS, &pCtx->cs);
4498 AssertRCReturn(rc, rc);
4499 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
4500 }
4501
4502 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
4503 {
4504 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
4505 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4506 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pCtx->ss.Attr.u;
4507 rc = HMVMX_EXPORT_SREG(SS, &pCtx->ss);
4508 AssertRCReturn(rc, rc);
4509 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
4510 }
4511
4512 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
4513 {
4514 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
4515 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4516 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pCtx->ds.Attr.u;
4517 rc = HMVMX_EXPORT_SREG(DS, &pCtx->ds);
4518 AssertRCReturn(rc, rc);
4519 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
4520 }
4521
4522 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
4523 {
4524 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
4525 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4526 pVCpu->hm.s.vmx.RealMode.AttrES.u = pCtx->es.Attr.u;
4527 rc = HMVMX_EXPORT_SREG(ES, &pCtx->es);
4528 AssertRCReturn(rc, rc);
4529 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
4530 }
4531
4532 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
4533 {
4534 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
4535 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4536 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pCtx->fs.Attr.u;
4537 rc = HMVMX_EXPORT_SREG(FS, &pCtx->fs);
4538 AssertRCReturn(rc, rc);
4539 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
4540 }
4541
4542 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
4543 {
4544 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
4545 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4546 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pCtx->gs.Attr.u;
4547 rc = HMVMX_EXPORT_SREG(GS, &pCtx->gs);
4548 AssertRCReturn(rc, rc);
4549 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
4550 }
4551
4552#ifdef VBOX_STRICT
4553 hmR0VmxValidateSegmentRegs(pVCpu);
4554#endif
4555
4556 Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pCtx->cs.Sel, pCtx->cs.u64Base,
4557 pCtx->cs.u32Limit, pCtx->cs.Attr.u));
4558 }
4559
4560 /*
4561 * Guest TR.
4562 */
4563 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4564 {
4565 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
4566
4567 /*
4568 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
4569 * achieved using the interrupt redirection bitmap (all bits cleared to let the guest
4570 * handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4571 */
4572 uint16_t u16Sel = 0;
4573 uint32_t u32Limit = 0;
4574 uint64_t u64Base = 0;
4575 uint32_t u32AccessRights = 0;
4576
4577 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4578 {
4579 u16Sel = pCtx->tr.Sel;
4580 u32Limit = pCtx->tr.u32Limit;
4581 u64Base = pCtx->tr.u64Base;
4582 u32AccessRights = pCtx->tr.Attr.u;
4583 }
4584 else
4585 {
4586 Assert(pVM->hm.s.vmx.pRealModeTSS);
4587 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
4588
4589 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4590 RTGCPHYS GCPhys;
4591 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4592 AssertRCReturn(rc, rc);
4593
4594 X86DESCATTR DescAttr;
4595 DescAttr.u = 0;
4596 DescAttr.n.u1Present = 1;
4597 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4598
4599 u16Sel = 0;
4600 u32Limit = HM_VTX_TSS_SIZE;
4601 u64Base = GCPhys; /* in real-mode phys = virt. */
4602 u32AccessRights = DescAttr.u;
4603 }
4604
4605 /* Validate. */
4606 Assert(!(u16Sel & RT_BIT(2)));
4607 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4608 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4609 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4610 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4611 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4612 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4613 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4614 Assert( (u32Limit & 0xfff) == 0xfff
4615 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4616 Assert( !(pCtx->tr.u32Limit & 0xfff00000)
4617 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4618
4619 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4620 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4621 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4622 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4623 AssertRCReturn(rc, rc);
4624
4625 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4626 Log4Func(("TR base=%#RX64\n", pCtx->tr.u64Base));
4627 }
4628
4629 /*
4630 * Guest GDTR.
4631 */
4632 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4633 {
4634 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
4635
4636 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
4637 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
4638 AssertRCReturn(rc, rc);
4639
4640 /* Validate. */
4641 Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4642
4643 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4644 Log4Func(("GDTR base=%#RX64\n", pCtx->gdtr.pGdt));
4645 }
4646
4647 /*
4648 * Guest LDTR.
4649 */
4650 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4651 {
4652 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
4653
4654 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4655 uint32_t u32Access = 0;
4656 if (!pCtx->ldtr.Attr.u)
4657 u32Access = X86DESCATTR_UNUSABLE;
4658 else
4659 u32Access = pCtx->ldtr.Attr.u;
4660
4661 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
4662 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
4663 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4664 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
4665 AssertRCReturn(rc, rc);
4666
4667 /* Validate. */
4668 if (!(u32Access & X86DESCATTR_UNUSABLE))
4669 {
4670 Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4671 Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4672 Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4673 Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4674 Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4675 Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4676 Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
4677 || !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4678 Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
4679 || pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4680 }
4681
4682 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4683 Log4Func(("LDTR base=%#RX64\n", pCtx->ldtr.u64Base));
4684 }
4685
4686 /*
4687 * Guest IDTR.
4688 */
4689 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4690 {
4691 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
4692
4693 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
4694 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
4695 AssertRCReturn(rc, rc);
4696
4697 /* Validate. */
4698 Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4699
4700 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4701 Log4Func(("IDTR base=%#RX64\n", pCtx->idtr.pIdt));
4702 }
4703
4704 return VINF_SUCCESS;
4705}
4706
4707
4708/**
4709 * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4710 * areas.
4711 *
4712 * These MSRs will automatically be loaded to the host CPU on every successful
4713 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4714 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4715 * -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4716 *
4717 * Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4718 *
4719 * @returns VBox status code.
4720 * @param pVCpu The cross context virtual CPU structure.
4721 *
4722 * @remarks No-long-jump zone!!!
4723 */
4724static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu)
4725{
4726 AssertPtr(pVCpu);
4727 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4728
4729 /*
4730 * MSRs that we use the auto-load/store MSR area in the VMCS.
4731 * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4732 */
4733 PVM pVM = pVCpu->CTX_SUFF(pVM);
4734 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4735 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4736 {
4737 if (pVM->hm.s.fAllow64BitGuests)
4738 {
4739#if HC_ARCH_BITS == 32
4740 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS | CPUMCTX_EXTRN_KERNEL_GS_BASE);
4741
4742 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pCtx->msrLSTAR, false, NULL);
4743 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pCtx->msrSTAR, false, NULL);
4744 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pCtx->msrSFMASK, false, NULL);
4745 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, false, NULL);
4746 AssertRCReturn(rc, rc);
4747# ifdef LOG_ENABLED
4748 PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4749 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4750 Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4751# endif
4752#endif
4753 }
4754 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4755 }
4756
4757 /*
4758 * Guest Sysenter MSRs.
4759 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4760 * VM-exits on WRMSRs for these MSRs.
4761 */
4762 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4763 {
4764 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
4765
4766 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4767 {
4768 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
4769 AssertRCReturn(rc, rc);
4770 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4771 }
4772
4773 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4774 {
4775 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
4776 AssertRCReturn(rc, rc);
4777 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4778 }
4779
4780 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4781 {
4782 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
4783 AssertRCReturn(rc, rc);
4784 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4785 }
4786 }
4787
4788 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4789 {
4790 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
4791
4792 if (hmR0VmxShouldSwapEferMsr(pVCpu))
4793 {
4794 /*
4795 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4796 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4797 */
4798 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4799 {
4800 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
4801 AssertRCReturn(rc,rc);
4802 Log4Func(("EFER=%#RX64\n", pCtx->msrEFER));
4803 }
4804 else
4805 {
4806 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pCtx->msrEFER, false /* fUpdateHostMsr */,
4807 NULL /* pfAddedAndUpdated */);
4808 AssertRCReturn(rc, rc);
4809
4810 /* We need to intercept reads too, see @bugref{7386#c16}. */
4811 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
4812 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4813 Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pCtx->msrEFER,
4814 pVCpu->hm.s.vmx.cMsrs));
4815 }
4816 }
4817 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4818 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4819 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4820 }
4821
4822 return VINF_SUCCESS;
4823}
4824
4825
4826#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4827/**
4828 * Check if guest state allows safe use of 32-bit switcher again.
4829 *
4830 * Segment bases and protected mode structures must be 32-bit addressable
4831 * because the 32-bit switcher will ignore high dword when writing these VMCS
4832 * fields. See @bugref{8432} for details.
4833 *
4834 * @returns true if safe, false if must continue to use the 64-bit switcher.
4835 * @param pCtx Pointer to the guest-CPU context.
4836 *
4837 * @remarks No-long-jump zone!!!
4838 */
4839static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
4840{
4841 if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4842 if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4843 if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4844 if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4845 if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4846 if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4847 if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4848 if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4849 if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4850 if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4851
4852 /* All good, bases are 32-bit. */
4853 return true;
4854}
4855#endif
4856
4857
4858/**
4859 * Selects up the appropriate function to run guest code.
4860 *
4861 * @returns VBox status code.
4862 * @param pVCpu The cross context virtual CPU structure.
4863 *
4864 * @remarks No-long-jump zone!!!
4865 */
4866static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu)
4867{
4868 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4869 if (CPUMIsGuestInLongModeEx(pCtx))
4870 {
4871#ifndef VBOX_ENABLE_64_BITS_GUESTS
4872 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4873#endif
4874 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4875#if HC_ARCH_BITS == 32
4876 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4877 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4878 {
4879#ifdef VBOX_STRICT
4880 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4881 {
4882 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4883 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4884 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4885 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4886 | HM_CHANGED_VMX_ENTRY_CTLS
4887 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4888 }
4889#endif
4890 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4891
4892 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4893 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4894 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4895 Log4Func(("Selected 64-bit switcher\n"));
4896 }
4897#else
4898 /* 64-bit host. */
4899 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4900#endif
4901 }
4902 else
4903 {
4904 /* Guest is not in long mode, use the 32-bit handler. */
4905#if HC_ARCH_BITS == 32
4906 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4907 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4908 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4909 {
4910# ifdef VBOX_STRICT
4911 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4912 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4913 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4914 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4915 | HM_CHANGED_VMX_ENTRY_CTLS
4916 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4917# endif
4918 }
4919# ifdef VBOX_ENABLE_64_BITS_GUESTS
4920 /*
4921 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4922 * design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4923 * switcher flag because now we know the guest is in a sane state where it's safe
4924 * to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4925 * the much faster 32-bit switcher again.
4926 */
4927 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4928 {
4929 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4930 Log4Func(("Selected 32-bit switcher\n"));
4931 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4932 }
4933 else
4934 {
4935 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4936 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4937 || hmR0VmxIs32BitSwitcherSafe(pCtx))
4938 {
4939 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4940 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4941 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4942 | HM_CHANGED_VMX_ENTRY_CTLS
4943 | HM_CHANGED_VMX_EXIT_CTLS
4944 | HM_CHANGED_HOST_CONTEXT);
4945 Log4Func(("Selected 32-bit switcher (safe)\n"));
4946 }
4947 }
4948# else
4949 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4950# endif
4951#else
4952 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4953#endif
4954 }
4955 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4956 return VINF_SUCCESS;
4957}
4958
4959
4960/**
4961 * Wrapper for running the guest code in VT-x.
4962 *
4963 * @returns VBox status code, no informational status codes.
4964 * @param pVCpu The cross context virtual CPU structure.
4965 *
4966 * @remarks No-long-jump zone!!!
4967 */
4968DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu)
4969{
4970 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4971 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4972 pCtx->fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
4973
4974 /*
4975 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4976 * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4977 * callee-saved and thus the need for this XMM wrapper.
4978 *
4979 * See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4980 */
4981 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4982 /** @todo Add stats for resume vs launch. */
4983 PVM pVM = pVCpu->CTX_SUFF(pVM);
4984#ifdef VBOX_WITH_KERNEL_USING_XMM
4985 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4986#else
4987 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4988#endif
4989 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4990 return rc;
4991}
4992
4993
4994/**
4995 * Reports world-switch error and dumps some useful debug info.
4996 *
4997 * @param pVCpu The cross context virtual CPU structure.
4998 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4999 * @param pVmxTransient Pointer to the VMX transient structure (only
5000 * exitReason updated).
5001 */
5002static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
5003{
5004 Assert(pVCpu);
5005 Assert(pVmxTransient);
5006 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
5007
5008 Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
5009 switch (rcVMRun)
5010 {
5011 case VERR_VMX_INVALID_VMXON_PTR:
5012 AssertFailed();
5013 break;
5014 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5015 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5016 {
5017 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5018 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5019 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
5020 AssertRC(rc);
5021
5022 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5023 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5024 Cannot do it here as we may have been long preempted. */
5025
5026#ifdef VBOX_STRICT
5027 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5028 pVmxTransient->uExitReason));
5029 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
5030 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5031 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5032 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5033 else
5034 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5035 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5036 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5037
5038 /* VMX control bits. */
5039 uint32_t u32Val;
5040 uint64_t u64Val;
5041 RTHCUINTREG uHCReg;
5042 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5043 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5044 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5045 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5046 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
5047 {
5048 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5049 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5050 }
5051 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5052 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5053 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5054 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5055 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5056 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5057 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5058 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5059 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5060 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5061 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5062 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5063 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5064 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5065 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5066 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5067 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5068 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5069 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5070 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5071 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5072 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5073 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5074 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5075 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5076 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5077 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5078 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5079 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5080 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5081 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5082 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5083 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5084 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5085 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5086 {
5087 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5088 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5089 }
5090
5091 /* Guest bits. */
5092 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5093 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
5094 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5095 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
5096 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5097 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
5098 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
5099 {
5100 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5101 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5102 }
5103
5104 /* Host bits. */
5105 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5106 Log4(("Host CR0 %#RHr\n", uHCReg));
5107 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5108 Log4(("Host CR3 %#RHr\n", uHCReg));
5109 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5110 Log4(("Host CR4 %#RHr\n", uHCReg));
5111
5112 RTGDTR HostGdtr;
5113 PCX86DESCHC pDesc;
5114 ASMGetGDTR(&HostGdtr);
5115 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5116 Log4(("Host CS %#08x\n", u32Val));
5117 if (u32Val < HostGdtr.cbGdt)
5118 {
5119 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5120 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5121 }
5122
5123 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5124 Log4(("Host DS %#08x\n", u32Val));
5125 if (u32Val < HostGdtr.cbGdt)
5126 {
5127 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5128 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5129 }
5130
5131 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5132 Log4(("Host ES %#08x\n", u32Val));
5133 if (u32Val < HostGdtr.cbGdt)
5134 {
5135 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5136 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5137 }
5138
5139 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5140 Log4(("Host FS %#08x\n", u32Val));
5141 if (u32Val < HostGdtr.cbGdt)
5142 {
5143 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5144 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5145 }
5146
5147 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5148 Log4(("Host GS %#08x\n", u32Val));
5149 if (u32Val < HostGdtr.cbGdt)
5150 {
5151 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5152 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5153 }
5154
5155 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5156 Log4(("Host SS %#08x\n", u32Val));
5157 if (u32Val < HostGdtr.cbGdt)
5158 {
5159 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5160 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5161 }
5162
5163 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5164 Log4(("Host TR %#08x\n", u32Val));
5165 if (u32Val < HostGdtr.cbGdt)
5166 {
5167 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5168 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5169 }
5170
5171 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5172 Log4(("Host TR Base %#RHv\n", uHCReg));
5173 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5174 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5175 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5176 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5177 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5178 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5179 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5180 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5181 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5182 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5183 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5184 Log4(("Host RSP %#RHv\n", uHCReg));
5185 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5186 Log4(("Host RIP %#RHv\n", uHCReg));
5187# if HC_ARCH_BITS == 64
5188 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5189 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5190 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5191 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5192 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5193 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5194# endif
5195#endif /* VBOX_STRICT */
5196 break;
5197 }
5198
5199 default:
5200 /* Impossible */
5201 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5202 break;
5203 }
5204}
5205
5206
5207#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5208#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5209# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5210#endif
5211#ifdef VBOX_STRICT
5212static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5213{
5214 switch (idxField)
5215 {
5216 case VMX_VMCS_GUEST_RIP:
5217 case VMX_VMCS_GUEST_RSP:
5218 case VMX_VMCS_GUEST_SYSENTER_EIP:
5219 case VMX_VMCS_GUEST_SYSENTER_ESP:
5220 case VMX_VMCS_GUEST_GDTR_BASE:
5221 case VMX_VMCS_GUEST_IDTR_BASE:
5222 case VMX_VMCS_GUEST_CS_BASE:
5223 case VMX_VMCS_GUEST_DS_BASE:
5224 case VMX_VMCS_GUEST_ES_BASE:
5225 case VMX_VMCS_GUEST_FS_BASE:
5226 case VMX_VMCS_GUEST_GS_BASE:
5227 case VMX_VMCS_GUEST_SS_BASE:
5228 case VMX_VMCS_GUEST_LDTR_BASE:
5229 case VMX_VMCS_GUEST_TR_BASE:
5230 case VMX_VMCS_GUEST_CR3:
5231 return true;
5232 }
5233 return false;
5234}
5235
5236static bool hmR0VmxIsValidReadField(uint32_t idxField)
5237{
5238 switch (idxField)
5239 {
5240 /* Read-only fields. */
5241 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5242 return true;
5243 }
5244 /* Remaining readable fields should also be writable. */
5245 return hmR0VmxIsValidWriteField(idxField);
5246}
5247#endif /* VBOX_STRICT */
5248
5249
5250/**
5251 * Executes the specified handler in 64-bit mode.
5252 *
5253 * @returns VBox status code (no informational status codes).
5254 * @param pVCpu The cross context virtual CPU structure.
5255 * @param enmOp The operation to perform.
5256 * @param cParams Number of parameters.
5257 * @param paParam Array of 32-bit parameters.
5258 */
5259VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
5260{
5261 PVM pVM = pVCpu->CTX_SUFF(pVM);
5262 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5263 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5264 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5265 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5266
5267#ifdef VBOX_STRICT
5268 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5269 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5270
5271 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5272 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5273#endif
5274
5275 /* Disable interrupts. */
5276 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5277
5278#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5279 RTCPUID idHostCpu = RTMpCpuId();
5280 CPUMR0SetLApic(pVCpu, idHostCpu);
5281#endif
5282
5283 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5284 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5285
5286 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5287 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5288 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5289
5290 /* Leave VMX Root Mode. */
5291 VMXDisable();
5292
5293 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5294
5295 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5296 CPUMSetHyperEIP(pVCpu, enmOp);
5297 for (int i = (int)cParams - 1; i >= 0; i--)
5298 CPUMPushHyper(pVCpu, paParam[i]);
5299
5300 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5301
5302 /* Call the switcher. */
5303 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
5304 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5305
5306 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5307 /* Make sure the VMX instructions don't cause #UD faults. */
5308 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5309
5310 /* Re-enter VMX Root Mode */
5311 int rc2 = VMXEnable(HCPhysCpuPage);
5312 if (RT_FAILURE(rc2))
5313 {
5314 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5315 ASMSetFlags(fOldEFlags);
5316 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5317 return rc2;
5318 }
5319
5320 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5321 AssertRC(rc2);
5322 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5323 Assert(!(ASMGetFlags() & X86_EFL_IF));
5324 ASMSetFlags(fOldEFlags);
5325 return rc;
5326}
5327
5328
5329/**
5330 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5331 * supporting 64-bit guests.
5332 *
5333 * @returns VBox status code.
5334 * @param fResume Whether to VMLAUNCH or VMRESUME.
5335 * @param pCtx Pointer to the guest-CPU context.
5336 * @param pCache Pointer to the VMCS cache.
5337 * @param pVM The cross context VM structure.
5338 * @param pVCpu The cross context virtual CPU structure.
5339 */
5340DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5341{
5342 NOREF(fResume);
5343
5344 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5345 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5346
5347#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5348 pCache->uPos = 1;
5349 pCache->interPD = PGMGetInterPaeCR3(pVM);
5350 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5351#endif
5352
5353#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5354 pCache->TestIn.HCPhysCpuPage = 0;
5355 pCache->TestIn.HCPhysVmcs = 0;
5356 pCache->TestIn.pCache = 0;
5357 pCache->TestOut.HCPhysVmcs = 0;
5358 pCache->TestOut.pCache = 0;
5359 pCache->TestOut.pCtx = 0;
5360 pCache->TestOut.eflags = 0;
5361#else
5362 NOREF(pCache);
5363#endif
5364
5365 uint32_t aParam[10];
5366 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5367 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5368 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5369 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5370 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5371 aParam[5] = 0;
5372 aParam[6] = VM_RC_ADDR(pVM, pVM);
5373 aParam[7] = 0;
5374 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5375 aParam[9] = 0;
5376
5377#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5378 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5379 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5380#endif
5381 int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5382
5383#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5384 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5385 Assert(pCtx->dr[4] == 10);
5386 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5387#endif
5388
5389#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5390 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5391 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5392 pVCpu->hm.s.vmx.HCPhysVmcs));
5393 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5394 pCache->TestOut.HCPhysVmcs));
5395 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5396 pCache->TestOut.pCache));
5397 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5398 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5399 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5400 pCache->TestOut.pCtx));
5401 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5402#endif
5403 NOREF(pCtx);
5404 return rc;
5405}
5406
5407
5408/**
5409 * Initialize the VMCS-Read cache.
5410 *
5411 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5412 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5413 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5414 * (those that have a 32-bit FULL & HIGH part).
5415 *
5416 * @returns VBox status code.
5417 * @param pVCpu The cross context virtual CPU structure.
5418 */
5419static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
5420{
5421#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5422 do { \
5423 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5424 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5425 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5426 ++cReadFields; \
5427 } while (0)
5428
5429 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5430 uint32_t cReadFields = 0;
5431
5432 /*
5433 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5434 * and serve to indicate exceptions to the rules.
5435 */
5436
5437 /* Guest-natural selector base fields. */
5438#if 0
5439 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5440 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5441 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5442#endif
5443 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5444 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5445 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5446 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5447 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5448 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5449 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5450 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5451 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5452 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5453 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5454 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5455#if 0
5456 /* Unused natural width guest-state fields. */
5457 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS);
5458 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5459#endif
5460 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5461 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5462
5463 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5464 these 64-bit fields (using "FULL" and "HIGH" fields). */
5465#if 0
5466 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5467 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5468 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5469 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5470 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5471 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5472 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5473 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5474 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5475#endif
5476
5477 /* Natural width guest-state fields. */
5478 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5479 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_GUEST_LINEAR_ADDR);
5480
5481 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5482 {
5483 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5484 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5485 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5486 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5487 }
5488 else
5489 {
5490 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5491 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5492 }
5493
5494#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5495 return VINF_SUCCESS;
5496}
5497
5498
5499/**
5500 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5501 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5502 * darwin, running 64-bit guests).
5503 *
5504 * @returns VBox status code.
5505 * @param pVCpu The cross context virtual CPU structure.
5506 * @param idxField The VMCS field encoding.
5507 * @param u64Val 16, 32 or 64-bit value.
5508 */
5509VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5510{
5511 int rc;
5512 switch (idxField)
5513 {
5514 /*
5515 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5516 */
5517 /* 64-bit Control fields. */
5518 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5519 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5520 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5521 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5522 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5523 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5524 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5525 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5526 case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
5527 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5528 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5529 case VMX_VMCS64_CTRL_EPTP_FULL:
5530 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5531 /* 64-bit Guest-state fields. */
5532 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5533 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5534 case VMX_VMCS64_GUEST_PAT_FULL:
5535 case VMX_VMCS64_GUEST_EFER_FULL:
5536 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5537 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5538 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5539 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5540 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5541 /* 64-bit Host-state fields. */
5542 case VMX_VMCS64_HOST_PAT_FULL:
5543 case VMX_VMCS64_HOST_EFER_FULL:
5544 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5545 {
5546 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5547 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5548 break;
5549 }
5550
5551 /*
5552 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5553 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5554 */
5555 /* Natural-width Guest-state fields. */
5556 case VMX_VMCS_GUEST_CR3:
5557 case VMX_VMCS_GUEST_ES_BASE:
5558 case VMX_VMCS_GUEST_CS_BASE:
5559 case VMX_VMCS_GUEST_SS_BASE:
5560 case VMX_VMCS_GUEST_DS_BASE:
5561 case VMX_VMCS_GUEST_FS_BASE:
5562 case VMX_VMCS_GUEST_GS_BASE:
5563 case VMX_VMCS_GUEST_LDTR_BASE:
5564 case VMX_VMCS_GUEST_TR_BASE:
5565 case VMX_VMCS_GUEST_GDTR_BASE:
5566 case VMX_VMCS_GUEST_IDTR_BASE:
5567 case VMX_VMCS_GUEST_RSP:
5568 case VMX_VMCS_GUEST_RIP:
5569 case VMX_VMCS_GUEST_SYSENTER_ESP:
5570 case VMX_VMCS_GUEST_SYSENTER_EIP:
5571 {
5572 if (!(RT_HI_U32(u64Val)))
5573 {
5574 /* If this field is 64-bit, VT-x will zero out the top bits. */
5575 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5576 }
5577 else
5578 {
5579 /* Assert that only the 32->64 switcher case should ever come here. */
5580 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5581 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5582 }
5583 break;
5584 }
5585
5586 default:
5587 {
5588 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5589 rc = VERR_INVALID_PARAMETER;
5590 break;
5591 }
5592 }
5593 AssertRCReturn(rc, rc);
5594 return rc;
5595}
5596
5597
5598/**
5599 * Queue up a VMWRITE by using the VMCS write cache.
5600 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5601 *
5602 * @param pVCpu The cross context virtual CPU structure.
5603 * @param idxField The VMCS field encoding.
5604 * @param u64Val 16, 32 or 64-bit value.
5605 */
5606VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5607{
5608 AssertPtr(pVCpu);
5609 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5610
5611 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5612 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5613
5614 /* Make sure there are no duplicates. */
5615 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5616 {
5617 if (pCache->Write.aField[i] == idxField)
5618 {
5619 pCache->Write.aFieldVal[i] = u64Val;
5620 return VINF_SUCCESS;
5621 }
5622 }
5623
5624 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5625 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5626 pCache->Write.cValidEntries++;
5627 return VINF_SUCCESS;
5628}
5629#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5630
5631
5632/**
5633 * Sets up the usage of TSC-offsetting and updates the VMCS.
5634 *
5635 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5636 * VMX preemption timer.
5637 *
5638 * @returns VBox status code.
5639 * @param pVCpu The cross context virtual CPU structure.
5640 *
5641 * @remarks No-long-jump zone!!!
5642 */
5643static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5644{
5645 bool fOffsettedTsc;
5646 bool fParavirtTsc;
5647 PVM pVM = pVCpu->CTX_SUFF(pVM);
5648 uint64_t uTscOffset;
5649 if (pVM->hm.s.vmx.fUsePreemptTimer)
5650 {
5651 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
5652
5653 /* Make sure the returned values have sane upper and lower boundaries. */
5654 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5655 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5656 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5657 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5658
5659 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5660 int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
5661 AssertRC(rc);
5662 }
5663 else
5664 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
5665
5666 if (fParavirtTsc)
5667 {
5668 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5669 information before every VM-entry, hence disable it for performance sake. */
5670#if 0
5671 int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5672 AssertRC(rc);
5673#endif
5674 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5675 }
5676
5677 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
5678 if ( fOffsettedTsc
5679 && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5680 {
5681 if (pVCpu->hm.s.vmx.u64TscOffset != uTscOffset)
5682 {
5683 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
5684 AssertRC(rc);
5685 pVCpu->hm.s.vmx.u64TscOffset = uTscOffset;
5686 }
5687
5688 if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
5689 {
5690 uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
5691 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5692 AssertRC(rc);
5693 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5694 }
5695 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5696 }
5697 else
5698 {
5699 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5700 if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
5701 {
5702 uProcCtls |= VMX_PROC_CTLS_RDTSC_EXIT;
5703 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5704 AssertRC(rc);
5705 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5706 }
5707 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5708 }
5709}
5710
5711
5712/**
5713 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5714 * VM-exit interruption info type.
5715 *
5716 * @returns The IEM exception flags.
5717 * @param uVector The event vector.
5718 * @param uVmxVectorType The VMX event type.
5719 *
5720 * @remarks This function currently only constructs flags required for
5721 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5722 * and CR2 aspects of an exception are not included).
5723 */
5724static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5725{
5726 uint32_t fIemXcptFlags;
5727 switch (uVmxVectorType)
5728 {
5729 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5730 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5731 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5732 break;
5733
5734 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5735 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5736 break;
5737
5738 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5739 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5740 break;
5741
5742 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5743 {
5744 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5745 if (uVector == X86_XCPT_BP)
5746 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5747 else if (uVector == X86_XCPT_OF)
5748 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5749 else
5750 {
5751 fIemXcptFlags = 0;
5752 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5753 }
5754 break;
5755 }
5756
5757 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5758 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5759 break;
5760
5761 default:
5762 fIemXcptFlags = 0;
5763 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5764 break;
5765 }
5766 return fIemXcptFlags;
5767}
5768
5769
5770/**
5771 * Sets an event as a pending event to be injected into the guest.
5772 *
5773 * @param pVCpu The cross context virtual CPU structure.
5774 * @param u32IntInfo The VM-entry interruption-information field.
5775 * @param cbInstr The VM-entry instruction length in bytes (for software
5776 * interrupts, exceptions and privileged software
5777 * exceptions).
5778 * @param u32ErrCode The VM-entry exception error code.
5779 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5780 * page-fault.
5781 *
5782 * @remarks Statistics counter assumes this is a guest event being injected or
5783 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5784 * always incremented.
5785 */
5786DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5787 RTGCUINTPTR GCPtrFaultAddress)
5788{
5789 Assert(!pVCpu->hm.s.Event.fPending);
5790 pVCpu->hm.s.Event.fPending = true;
5791 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5792 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5793 pVCpu->hm.s.Event.cbInstr = cbInstr;
5794 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5795}
5796
5797
5798/**
5799 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5800 *
5801 * @param pVCpu The cross context virtual CPU structure.
5802 */
5803DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
5804{
5805 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
5806 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5807 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5808 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5809 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5810}
5811
5812
5813/**
5814 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
5815 *
5816 * @param pVCpu The cross context virtual CPU structure.
5817 */
5818DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
5819{
5820 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
5821 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5822 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
5823 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5824 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5825}
5826
5827
5828/**
5829 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
5830 *
5831 * @param pVCpu The cross context virtual CPU structure.
5832 */
5833DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
5834{
5835 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
5836 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5837 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
5838 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5839 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5840}
5841
5842
5843#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5844/**
5845 * Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
5846 *
5847 * @param pVCpu The cross context virtual CPU structure.
5848 * @param u32ErrCode The error code for the general-protection exception.
5849 */
5850DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
5851{
5852 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
5853 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5854 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5855 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5856 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrCode, 0 /* GCPtrFaultAddress */);
5857}
5858
5859
5860/**
5861 * Sets a stack (\#SS) exception as pending-for-injection into the VM.
5862 *
5863 * @param pVCpu The cross context virtual CPU structure.
5864 * @param u32ErrCode The error code for the stack exception.
5865 */
5866DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
5867{
5868 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
5869 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5870 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5871 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5872 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrCode, 0 /* GCPtrFaultAddress */);
5873}
5874
5875
5876/**
5877 * Decodes the memory operand of an instruction that caused a VM-exit.
5878 *
5879 * The VM-exit qualification field provides the displacement field for memory
5880 * operand instructions, if any.
5881 *
5882 * @returns Strict VBox status code (i.e. informational status codes too).
5883 * @retval VINF_SUCCESS if the operand was successfully decoded.
5884 * @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
5885 * operand.
5886 * @param pVCpu The cross context virtual CPU structure.
5887 * @param uExitInstrInfo The VM-exit instruction information field.
5888 * @param enmMemAccess The memory operand's access type (read or write).
5889 * @param GCPtrDisp The instruction displacement field, if any. For
5890 * RIP-relative addressing pass RIP + displacement here.
5891 * @param pGCPtrMem Where to store the effective destination memory address.
5892 */
5893static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
5894 PRTGCPTR pGCPtrMem)
5895{
5896 Assert(pGCPtrMem);
5897 Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
5898 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_EFER | CPUMCTX_EXTRN_CR0);
5899
5900 static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
5901 static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
5902 AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
5903
5904 VMXEXITINSTRINFO ExitInstrInfo;
5905 ExitInstrInfo.u = uExitInstrInfo;
5906 uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
5907 uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
5908 bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
5909 uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
5910 uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
5911 bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
5912 uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
5913 bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
5914 bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
5915
5916 /*
5917 * Validate instruction information.
5918 * This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
5919 */
5920 AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
5921 ("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
5922 AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
5923 ("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
5924 AssertLogRelMsgReturn(fIsMemOperand,
5925 ("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
5926
5927 /*
5928 * Compute the complete effective address.
5929 *
5930 * See AMD instruction spec. 1.4.2 "SIB Byte Format"
5931 * See AMD spec. 4.5.2 "Segment Registers".
5932 */
5933 RTGCPTR GCPtrMem = GCPtrDisp;
5934 if (fBaseRegValid)
5935 GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
5936 if (fIdxRegValid)
5937 GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
5938
5939 RTGCPTR const GCPtrOff = GCPtrMem;
5940 if ( !fIsLongMode
5941 || iSegReg >= X86_SREG_FS)
5942 GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
5943 GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
5944
5945 /*
5946 * Validate effective address.
5947 * See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
5948 */
5949 uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
5950 Assert(cbAccess > 0);
5951 if (fIsLongMode)
5952 {
5953 if (X86_IS_CANONICAL(GCPtrMem))
5954 {
5955 *pGCPtrMem = GCPtrMem;
5956 return VINF_SUCCESS;
5957 }
5958
5959 /** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
5960 * "Data Limit Checks in 64-bit Mode". */
5961 Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
5962 hmR0VmxSetPendingXcptGP(pVCpu, 0);
5963 return VINF_HM_PENDING_XCPT;
5964 }
5965
5966 /*
5967 * This is a watered down version of iemMemApplySegment().
5968 * Parts that are not applicable for VMX instructions like real-or-v8086 mode
5969 * and segment CPL/DPL checks are skipped.
5970 */
5971 RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
5972 RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
5973 PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
5974
5975 /* Check if the segment is present and usable. */
5976 if ( pSel->Attr.n.u1Present
5977 && !pSel->Attr.n.u1Unusable)
5978 {
5979 Assert(pSel->Attr.n.u1DescType);
5980 if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
5981 {
5982 /* Check permissions for the data segment. */
5983 if ( enmMemAccess == VMXMEMACCESS_WRITE
5984 && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
5985 {
5986 Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
5987 hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
5988 return VINF_HM_PENDING_XCPT;
5989 }
5990
5991 /* Check limits if it's a normal data segment. */
5992 if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
5993 {
5994 if ( GCPtrFirst32 > pSel->u32Limit
5995 || GCPtrLast32 > pSel->u32Limit)
5996 {
5997 Log4Func(("Data segment limit exceeded."
5998 "iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
5999 GCPtrLast32, pSel->u32Limit));
6000 if (iSegReg == X86_SREG_SS)
6001 hmR0VmxSetPendingXcptSS(pVCpu, 0);
6002 else
6003 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6004 return VINF_HM_PENDING_XCPT;
6005 }
6006 }
6007 else
6008 {
6009 /* Check limits if it's an expand-down data segment.
6010 Note! The upper boundary is defined by the B bit, not the G bit! */
6011 if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
6012 || GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
6013 {
6014 Log4Func(("Expand-down data segment limit exceeded."
6015 "iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6016 GCPtrLast32, pSel->u32Limit));
6017 if (iSegReg == X86_SREG_SS)
6018 hmR0VmxSetPendingXcptSS(pVCpu, 0);
6019 else
6020 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6021 return VINF_HM_PENDING_XCPT;
6022 }
6023 }
6024 }
6025 else
6026 {
6027 /* Check permissions for the code segment. */
6028 if ( enmMemAccess == VMXMEMACCESS_WRITE
6029 || ( enmMemAccess == VMXMEMACCESS_READ
6030 && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
6031 {
6032 Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
6033 Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
6034 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6035 return VINF_HM_PENDING_XCPT;
6036 }
6037
6038 /* Check limits for the code segment (normal/expand-down not applicable for code segments). */
6039 if ( GCPtrFirst32 > pSel->u32Limit
6040 || GCPtrLast32 > pSel->u32Limit)
6041 {
6042 Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
6043 GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
6044 if (iSegReg == X86_SREG_SS)
6045 hmR0VmxSetPendingXcptSS(pVCpu, 0);
6046 else
6047 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6048 return VINF_HM_PENDING_XCPT;
6049 }
6050 }
6051 }
6052 else
6053 {
6054 Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6055 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6056 return VINF_HM_PENDING_XCPT;
6057 }
6058
6059 *pGCPtrMem = GCPtrMem;
6060 return VINF_SUCCESS;
6061}
6062
6063
6064/**
6065 * Perform the relevant VMX instruction checks for VM-exits that occurred due to the
6066 * guest attempting to execute a VMX instruction.
6067 *
6068 * @returns Strict VBox status code (i.e. informational status codes too).
6069 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
6070 * @retval VINF_HM_PENDING_XCPT if an exception was raised.
6071 *
6072 * @param pVCpu The cross context virtual CPU structure.
6073 * @param uExitReason The VM-exit reason.
6074 *
6075 * @todo NstVmx: Document other error codes when VM-exit is implemented.
6076 * @remarks No-long-jump zone!!!
6077 */
6078static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
6079{
6080 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS
6081 | CPUMCTX_EXTRN_HWVIRT);
6082
6083 if ( CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx)
6084 || ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6085 && !CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx)))
6086 {
6087 Log4Func(("In real/v86-mode or long-mode outside 64-bit code segment -> #UD\n"));
6088 hmR0VmxSetPendingXcptUD(pVCpu);
6089 return VINF_HM_PENDING_XCPT;
6090 }
6091
6092 if (uExitReason == VMX_EXIT_VMXON)
6093 {
6094 /*
6095 * We check CR4.VMXE because it is required to be always set while in VMX operation
6096 * by physical CPUs and our CR4 read shadow is only consulted when executing specific
6097 * instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
6098 * otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
6099 */
6100 if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
6101 {
6102 Log4Func(("CR4.VMXE is not set -> #UD\n"));
6103 hmR0VmxSetPendingXcptUD(pVCpu);
6104 return VINF_HM_PENDING_XCPT;
6105 }
6106 }
6107 else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
6108 {
6109 /*
6110 * The guest has not entered VMX operation but attempted to execute a VMX instruction
6111 * (other than VMXON), we need to raise a #UD.
6112 */
6113 Log4Func(("Not in VMX root mode -> #UD\n"));
6114 hmR0VmxSetPendingXcptUD(pVCpu);
6115 return VINF_HM_PENDING_XCPT;
6116 }
6117
6118 if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
6119 {
6120 /*
6121 * The nested-guest attempted to execute a VMX instruction, cause a VM-exit and let
6122 * the guest hypervisor deal with it.
6123 */
6124 /** @todo NSTVMX: Trigger a VM-exit */
6125 }
6126
6127 /*
6128 * VMX instructions require CPL 0 except in VMX non-root mode where the VM-exit intercept
6129 * (above) takes preceedence over the CPL check.
6130 */
6131 if (CPUMGetGuestCPL(pVCpu) > 0)
6132 {
6133 Log4Func(("CPL > 0 -> #GP(0)\n"));
6134 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6135 return VINF_HM_PENDING_XCPT;
6136 }
6137
6138 return VINF_SUCCESS;
6139}
6140
6141#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
6142
6143
6144/**
6145 * Handle a condition that occurred while delivering an event through the guest
6146 * IDT.
6147 *
6148 * @returns Strict VBox status code (i.e. informational status codes too).
6149 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
6150 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
6151 * to continue execution of the guest which will delivery the \#DF.
6152 * @retval VINF_EM_RESET if we detected a triple-fault condition.
6153 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
6154 *
6155 * @param pVCpu The cross context virtual CPU structure.
6156 * @param pVmxTransient Pointer to the VMX transient structure.
6157 *
6158 * @remarks No-long-jump zone!!!
6159 */
6160static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6161{
6162 uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
6163
6164 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
6165 rc2 |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
6166 AssertRCReturn(rc2, rc2);
6167
6168 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
6169 if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6170 {
6171 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
6172 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
6173
6174 /*
6175 * If the event was a software interrupt (generated with INT n) or a software exception
6176 * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
6177 * can handle the VM-exit and continue guest execution which will re-execute the
6178 * instruction rather than re-injecting the exception, as that can cause premature
6179 * trips to ring-3 before injection and involve TRPM which currently has no way of
6180 * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
6181 * the problem).
6182 */
6183 IEMXCPTRAISE enmRaise;
6184 IEMXCPTRAISEINFO fRaiseInfo;
6185 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6186 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6187 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6188 {
6189 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
6190 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6191 }
6192 else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
6193 {
6194 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
6195 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
6196 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
6197 /** @todo Make AssertMsgReturn as just AssertMsg later. */
6198 AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
6199 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
6200 uExitVectorType), VERR_VMX_IPE_5);
6201
6202 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
6203
6204 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
6205 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
6206 {
6207 pVmxTransient->fVectoringPF = true;
6208 enmRaise = IEMXCPTRAISE_PREV_EVENT;
6209 }
6210 }
6211 else
6212 {
6213 /*
6214 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
6215 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
6216 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
6217 */
6218 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6219 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6220 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
6221 enmRaise = IEMXCPTRAISE_PREV_EVENT;
6222 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6223 }
6224
6225 /*
6226 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
6227 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
6228 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
6229 * subsequent VM-entry would fail.
6230 *
6231 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
6232 */
6233 if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS)
6234 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6235 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
6236 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
6237 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
6238 {
6239 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6240 }
6241
6242 switch (enmRaise)
6243 {
6244 case IEMXCPTRAISE_CURRENT_XCPT:
6245 {
6246 Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
6247 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
6248 Assert(rcStrict == VINF_SUCCESS);
6249 break;
6250 }
6251
6252 case IEMXCPTRAISE_PREV_EVENT:
6253 {
6254 uint32_t u32ErrCode;
6255 if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
6256 {
6257 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6258 AssertRCReturn(rc2, rc2);
6259 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6260 }
6261 else
6262 u32ErrCode = 0;
6263
6264 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
6265 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6266 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6267 0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
6268
6269 Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
6270 pVCpu->hm.s.Event.u32ErrCode));
6271 Assert(rcStrict == VINF_SUCCESS);
6272 break;
6273 }
6274
6275 case IEMXCPTRAISE_REEXEC_INSTR:
6276 Assert(rcStrict == VINF_SUCCESS);
6277 break;
6278
6279 case IEMXCPTRAISE_DOUBLE_FAULT:
6280 {
6281 /*
6282 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
6283 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
6284 */
6285 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6286 {
6287 pVmxTransient->fVectoringDoublePF = true;
6288 Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
6289 pVCpu->cpum.GstCtx.cr2));
6290 rcStrict = VINF_SUCCESS;
6291 }
6292 else
6293 {
6294 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6295 hmR0VmxSetPendingXcptDF(pVCpu);
6296 Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
6297 uIdtVector, uExitVector));
6298 rcStrict = VINF_HM_DOUBLE_FAULT;
6299 }
6300 break;
6301 }
6302
6303 case IEMXCPTRAISE_TRIPLE_FAULT:
6304 {
6305 Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
6306 rcStrict = VINF_EM_RESET;
6307 break;
6308 }
6309
6310 case IEMXCPTRAISE_CPU_HANG:
6311 {
6312 Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
6313 rcStrict = VERR_EM_GUEST_CPU_HANG;
6314 break;
6315 }
6316
6317 default:
6318 {
6319 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6320 rcStrict = VERR_VMX_IPE_2;
6321 break;
6322 }
6323 }
6324 }
6325 else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6326 && VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6327 && uExitVector != X86_XCPT_DF
6328 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
6329 {
6330 /*
6331 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6332 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6333 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6334 */
6335 if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
6336 {
6337 Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
6338 VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6339 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6340 }
6341 }
6342
6343 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
6344 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
6345 return rcStrict;
6346}
6347
6348
6349/**
6350 * Imports a guest segment register from the current VMCS into
6351 * the guest-CPU context.
6352 *
6353 * @returns VBox status code.
6354 * @param pVCpu The cross context virtual CPU structure.
6355 * @param idxSel Index of the selector in the VMCS.
6356 * @param idxLimit Index of the segment limit in the VMCS.
6357 * @param idxBase Index of the segment base in the VMCS.
6358 * @param idxAccess Index of the access rights of the segment in the VMCS.
6359 * @param pSelReg Pointer to the segment selector.
6360 *
6361 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6362 * do not log!
6363 *
6364 * @remarks Never call this function directly!!! Use the
6365 * HMVMX_IMPORT_SREG() macro as that takes care
6366 * of whether to read from the VMCS cache or not.
6367 */
6368static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6369 PCPUMSELREG pSelReg)
6370{
6371 NOREF(pVCpu);
6372
6373 uint32_t u32Sel;
6374 uint32_t u32Limit;
6375 uint32_t u32Attr;
6376 uint64_t u64Base;
6377 int rc = VMXReadVmcs32(idxSel, &u32Sel);
6378 rc |= VMXReadVmcs32(idxLimit, &u32Limit);
6379 rc |= VMXReadVmcs32(idxAccess, &u32Attr);
6380 rc |= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
6381 AssertRCReturn(rc, rc);
6382
6383 pSelReg->Sel = (uint16_t)u32Sel;
6384 pSelReg->ValidSel = (uint16_t)u32Sel;
6385 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6386 pSelReg->u32Limit = u32Limit;
6387 pSelReg->u64Base = u64Base;
6388 pSelReg->Attr.u = u32Attr;
6389
6390 /*
6391 * If VT-x marks the segment as unusable, most other bits remain undefined:
6392 * - For CS the L, D and G bits have meaning.
6393 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6394 * - For the remaining data segments no bits are defined.
6395 *
6396 * The present bit and the unusable bit has been observed to be set at the
6397 * same time (the selector was supposed to be invalid as we started executing
6398 * a V8086 interrupt in ring-0).
6399 *
6400 * What should be important for the rest of the VBox code, is that the P bit is
6401 * cleared. Some of the other VBox code recognizes the unusable bit, but
6402 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6403 * safe side here, we'll strip off P and other bits we don't care about. If
6404 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6405 *
6406 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6407 */
6408 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6409 {
6410 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6411
6412 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6413 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6414 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6415#ifdef VBOX_STRICT
6416 VMMRZCallRing3Disable(pVCpu);
6417 Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
6418# ifdef DEBUG_bird
6419 AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
6420 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6421 idxSel, u32Sel, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6422# endif
6423 VMMRZCallRing3Enable(pVCpu);
6424#endif
6425 }
6426 return VINF_SUCCESS;
6427}
6428
6429
6430/**
6431 * Imports the guest RIP from the VMCS back into the guest-CPU context.
6432 *
6433 * @returns VBox status code.
6434 * @param pVCpu The cross context virtual CPU structure.
6435 *
6436 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6437 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6438 * instead!!!
6439 */
6440DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6441{
6442 uint64_t u64Val;
6443 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6444 if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6445 {
6446 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6447 if (RT_SUCCESS(rc))
6448 {
6449 pCtx->rip = u64Val;
6450 EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
6451 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6452 }
6453 return rc;
6454 }
6455 return VINF_SUCCESS;
6456}
6457
6458
6459/**
6460 * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6461 *
6462 * @returns VBox status code.
6463 * @param pVCpu The cross context virtual CPU structure.
6464 *
6465 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6466 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6467 * instead!!!
6468 */
6469DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6470{
6471 uint32_t u32Val;
6472 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6473 if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6474 {
6475 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6476 if (RT_SUCCESS(rc))
6477 {
6478 pCtx->eflags.u32 = u32Val;
6479
6480 /* Restore eflags for real-on-v86-mode hack. */
6481 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6482 {
6483 pCtx->eflags.Bits.u1VM = 0;
6484 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6485 }
6486 }
6487 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6488 return rc;
6489 }
6490 return VINF_SUCCESS;
6491}
6492
6493
6494/**
6495 * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6496 * context.
6497 *
6498 * @returns VBox status code.
6499 * @param pVCpu The cross context virtual CPU structure.
6500 *
6501 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6502 * do not log!
6503 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6504 * instead!!!
6505 */
6506DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6507{
6508 uint32_t u32Val;
6509 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6510 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
6511 AssertRCReturn(rc, rc);
6512
6513 /*
6514 * We additionally have a requirement to import RIP, RFLAGS depending on whether we
6515 * might need them in hmR0VmxEvaluatePendingEvent().
6516 */
6517 if (!u32Val)
6518 {
6519 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6520 {
6521 rc = hmR0VmxImportGuestRip(pVCpu);
6522 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6523 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6524 }
6525
6526 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
6527 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6528 }
6529 else
6530 {
6531 rc = hmR0VmxImportGuestRip(pVCpu);
6532 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6533
6534 if (u32Val & ( VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
6535 | VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
6536 {
6537 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6538 }
6539 else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6540 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6541
6542 if (u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
6543 {
6544 if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
6545 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6546 }
6547 else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
6548 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6549 }
6550
6551 return VINF_SUCCESS;
6552}
6553
6554
6555/**
6556 * Worker for VMXR0ImportStateOnDemand.
6557 *
6558 * @returns VBox status code.
6559 * @param pVCpu The cross context virtual CPU structure.
6560 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6561 */
6562static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6563{
6564#define VMXLOCAL_BREAK_RC(a_rc) \
6565 if (RT_FAILURE(a_rc)) \
6566 break
6567
6568 int rc = VINF_SUCCESS;
6569 PVM pVM = pVCpu->CTX_SUFF(pVM);
6570 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6571 uint64_t u64Val;
6572 uint32_t u32Val;
6573
6574 Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6575 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6576
6577 /*
6578 * We disable interrupts to make the updating of the state and in particular
6579 * the fExtrn modification atomic wrt to preemption hooks.
6580 */
6581 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6582
6583 fWhat &= pCtx->fExtrn;
6584 if (fWhat)
6585 {
6586 do
6587 {
6588 if (fWhat & CPUMCTX_EXTRN_RIP)
6589 {
6590 rc = hmR0VmxImportGuestRip(pVCpu);
6591 VMXLOCAL_BREAK_RC(rc);
6592 }
6593
6594 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6595 {
6596 rc = hmR0VmxImportGuestRFlags(pVCpu);
6597 VMXLOCAL_BREAK_RC(rc);
6598 }
6599
6600 if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6601 {
6602 rc = hmR0VmxImportGuestIntrState(pVCpu);
6603 VMXLOCAL_BREAK_RC(rc);
6604 }
6605
6606 if (fWhat & CPUMCTX_EXTRN_RSP)
6607 {
6608 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6609 VMXLOCAL_BREAK_RC(rc);
6610 pCtx->rsp = u64Val;
6611 }
6612
6613 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6614 {
6615 if (fWhat & CPUMCTX_EXTRN_CS)
6616 {
6617 rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6618 rc |= hmR0VmxImportGuestRip(pVCpu);
6619 VMXLOCAL_BREAK_RC(rc);
6620 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6621 pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6622 EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true);
6623 }
6624 if (fWhat & CPUMCTX_EXTRN_SS)
6625 {
6626 rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6627 VMXLOCAL_BREAK_RC(rc);
6628 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6629 pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6630 }
6631 if (fWhat & CPUMCTX_EXTRN_DS)
6632 {
6633 rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6634 VMXLOCAL_BREAK_RC(rc);
6635 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6636 pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6637 }
6638 if (fWhat & CPUMCTX_EXTRN_ES)
6639 {
6640 rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6641 VMXLOCAL_BREAK_RC(rc);
6642 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6643 pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6644 }
6645 if (fWhat & CPUMCTX_EXTRN_FS)
6646 {
6647 rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6648 VMXLOCAL_BREAK_RC(rc);
6649 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6650 pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6651 }
6652 if (fWhat & CPUMCTX_EXTRN_GS)
6653 {
6654 rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6655 VMXLOCAL_BREAK_RC(rc);
6656 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6657 pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6658 }
6659 }
6660
6661 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6662 {
6663 if (fWhat & CPUMCTX_EXTRN_LDTR)
6664 {
6665 rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6666 VMXLOCAL_BREAK_RC(rc);
6667 }
6668
6669 if (fWhat & CPUMCTX_EXTRN_GDTR)
6670 {
6671 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6672 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6673 VMXLOCAL_BREAK_RC(rc);
6674 pCtx->gdtr.pGdt = u64Val;
6675 pCtx->gdtr.cbGdt = u32Val;
6676 }
6677
6678 /* Guest IDTR. */
6679 if (fWhat & CPUMCTX_EXTRN_IDTR)
6680 {
6681 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6682 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6683 VMXLOCAL_BREAK_RC(rc);
6684 pCtx->idtr.pIdt = u64Val;
6685 pCtx->idtr.cbIdt = u32Val;
6686 }
6687
6688 /* Guest TR. */
6689 if (fWhat & CPUMCTX_EXTRN_TR)
6690 {
6691 /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6692 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6693 {
6694 rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6695 VMXLOCAL_BREAK_RC(rc);
6696 }
6697 }
6698 }
6699
6700 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6701 {
6702 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6703 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6704 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6705 pCtx->SysEnter.cs = u32Val;
6706 VMXLOCAL_BREAK_RC(rc);
6707 }
6708
6709#if HC_ARCH_BITS == 64
6710 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6711 {
6712 if ( pVM->hm.s.fAllow64BitGuests
6713 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6714 pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6715 }
6716
6717 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6718 {
6719 if ( pVM->hm.s.fAllow64BitGuests
6720 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6721 {
6722 pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6723 pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6724 pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6725 }
6726 }
6727#endif
6728
6729 if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
6730#if HC_ARCH_BITS == 32
6731 || (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS))
6732#endif
6733 )
6734 {
6735 PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6736 uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6737 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6738 {
6739 switch (pMsr->u32Msr)
6740 {
6741#if HC_ARCH_BITS == 32
6742 case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6743 case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6744 case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6745 case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6746#endif
6747 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6748 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6749 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit */ break;
6750 default:
6751 {
6752 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6753 ASMSetFlags(fEFlags);
6754 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6755 cMsrs));
6756 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6757 }
6758 }
6759 }
6760 }
6761
6762 if (fWhat & CPUMCTX_EXTRN_DR7)
6763 {
6764 if (!pVCpu->hm.s.fUsingHyperDR7)
6765 {
6766 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6767 rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6768 VMXLOCAL_BREAK_RC(rc);
6769 pCtx->dr[7] = u32Val;
6770 }
6771 }
6772
6773 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6774 {
6775 uint32_t u32Shadow;
6776 if (fWhat & CPUMCTX_EXTRN_CR0)
6777 {
6778 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6779 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6780 VMXLOCAL_BREAK_RC(rc);
6781 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr0Mask)
6782 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr0Mask);
6783 VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6784 CPUMSetGuestCR0(pVCpu, u32Val);
6785 VMMRZCallRing3Enable(pVCpu);
6786 }
6787
6788 if (fWhat & CPUMCTX_EXTRN_CR4)
6789 {
6790 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6791 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6792 VMXLOCAL_BREAK_RC(rc);
6793 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr4Mask)
6794 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr4Mask);
6795 CPUMSetGuestCR4(pVCpu, u32Val);
6796 }
6797
6798 if (fWhat & CPUMCTX_EXTRN_CR3)
6799 {
6800 /* CR0.PG bit changes are always intercepted, so it's up to date. */
6801 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6802 || ( pVM->hm.s.fNestedPaging
6803 && CPUMIsGuestPagingEnabledEx(pCtx)))
6804 {
6805 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6806 if (pCtx->cr3 != u64Val)
6807 {
6808 CPUMSetGuestCR3(pVCpu, u64Val);
6809 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6810 }
6811
6812 /* If the guest is in PAE mode, sync back the PDPE's into the guest state.
6813 Note: CR4.PAE, CR0.PG, EFER bit changes are always intercepted, so they're up to date. */
6814 if (CPUMIsGuestInPAEModeEx(pCtx))
6815 {
6816 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6817 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6818 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6819 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6820 VMXLOCAL_BREAK_RC(rc);
6821 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6822 }
6823 }
6824 }
6825 }
6826 } while (0);
6827
6828 if (RT_SUCCESS(rc))
6829 {
6830 /* Update fExtrn. */
6831 pCtx->fExtrn &= ~fWhat;
6832
6833 /* If everything has been imported, clear the HM keeper bit. */
6834 if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6835 {
6836 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6837 Assert(!pCtx->fExtrn);
6838 }
6839 }
6840 }
6841 else
6842 AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6843
6844 ASMSetFlags(fEFlags);
6845
6846 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6847
6848 /*
6849 * Honor any pending CR3 updates.
6850 *
6851 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6852 * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6853 * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6854 *
6855 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6856 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6857 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6858 * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6859 *
6860 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6861 */
6862 if (VMMRZCallRing3IsEnabled(pVCpu))
6863 {
6864 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6865 {
6866 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
6867 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6868 }
6869
6870 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6871 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6872
6873 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6874 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6875 }
6876
6877 return VINF_SUCCESS;
6878#undef VMXLOCAL_BREAK_RC
6879}
6880
6881
6882/**
6883 * Saves the guest state from the VMCS into the guest-CPU context.
6884 *
6885 * @returns VBox status code.
6886 * @param pVCpu The cross context virtual CPU structure.
6887 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6888 */
6889VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6890{
6891 return hmR0VmxImportGuestState(pVCpu, fWhat);
6892}
6893
6894
6895/**
6896 * Check per-VM and per-VCPU force flag actions that require us to go back to
6897 * ring-3 for one reason or another.
6898 *
6899 * @returns Strict VBox status code (i.e. informational status codes too)
6900 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6901 * ring-3.
6902 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6903 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6904 * interrupts)
6905 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6906 * all EMTs to be in ring-3.
6907 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6908 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6909 * to the EM loop.
6910 *
6911 * @param pVCpu The cross context virtual CPU structure.
6912 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6913 */
6914static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
6915{
6916 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6917
6918 /*
6919 * Anything pending? Should be more likely than not if we're doing a good job.
6920 */
6921 PVM pVM = pVCpu->CTX_SUFF(pVM);
6922 if ( !fStepping
6923 ? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6924 && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6925 : !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6926 && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6927 return VINF_SUCCESS;
6928
6929 /* Pending PGM C3 sync. */
6930 if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6931 {
6932 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6933 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
6934 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
6935 VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6936 if (rcStrict2 != VINF_SUCCESS)
6937 {
6938 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6939 Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6940 return rcStrict2;
6941 }
6942 }
6943
6944 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6945 if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
6946 || VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6947 {
6948 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6949 int rc2 = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
6950 Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6951 return rc2;
6952 }
6953
6954 /* Pending VM request packets, such as hardware interrupts. */
6955 if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
6956 || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
6957 {
6958 Log4Func(("Pending VM request forcing us back to ring-3\n"));
6959 return VINF_EM_PENDING_REQUEST;
6960 }
6961
6962 /* Pending PGM pool flushes. */
6963 if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6964 {
6965 Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6966 return VINF_PGM_POOL_FLUSH_PENDING;
6967 }
6968
6969 /* Pending DMA requests. */
6970 if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
6971 {
6972 Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6973 return VINF_EM_RAW_TO_R3;
6974 }
6975
6976 return VINF_SUCCESS;
6977}
6978
6979
6980/**
6981 * Converts any TRPM trap into a pending HM event. This is typically used when
6982 * entering from ring-3 (not longjmp returns).
6983 *
6984 * @param pVCpu The cross context virtual CPU structure.
6985 */
6986static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6987{
6988 Assert(TRPMHasTrap(pVCpu));
6989 Assert(!pVCpu->hm.s.Event.fPending);
6990
6991 uint8_t uVector;
6992 TRPMEVENT enmTrpmEvent;
6993 RTGCUINT uErrCode;
6994 RTGCUINTPTR GCPtrFaultAddress;
6995 uint8_t cbInstr;
6996
6997 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6998 AssertRC(rc);
6999
7000 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7001 uint32_t u32IntInfo = uVector | VMX_EXIT_INT_INFO_VALID;
7002 if (enmTrpmEvent == TRPM_TRAP)
7003 {
7004 switch (uVector)
7005 {
7006 case X86_XCPT_NMI:
7007 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7008 break;
7009
7010 case X86_XCPT_BP:
7011 case X86_XCPT_OF:
7012 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7013 break;
7014
7015 case X86_XCPT_PF:
7016 case X86_XCPT_DF:
7017 case X86_XCPT_TS:
7018 case X86_XCPT_NP:
7019 case X86_XCPT_SS:
7020 case X86_XCPT_GP:
7021 case X86_XCPT_AC:
7022 u32IntInfo |= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7023 RT_FALL_THRU();
7024 default:
7025 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7026 break;
7027 }
7028 }
7029 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7030 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7031 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7032 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7033 else
7034 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7035
7036 rc = TRPMResetTrap(pVCpu);
7037 AssertRC(rc);
7038 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7039 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7040
7041 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7042}
7043
7044
7045/**
7046 * Converts the pending HM event into a TRPM trap.
7047 *
7048 * @param pVCpu The cross context virtual CPU structure.
7049 */
7050static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7051{
7052 Assert(pVCpu->hm.s.Event.fPending);
7053
7054 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7055 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7056 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
7057 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7058
7059 /* If a trap was already pending, we did something wrong! */
7060 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7061
7062 TRPMEVENT enmTrapType;
7063 switch (uVectorType)
7064 {
7065 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7066 enmTrapType = TRPM_HARDWARE_INT;
7067 break;
7068
7069 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7070 enmTrapType = TRPM_SOFTWARE_INT;
7071 break;
7072
7073 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7074 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7075 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7076 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7077 enmTrapType = TRPM_TRAP;
7078 break;
7079
7080 default:
7081 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7082 enmTrapType = TRPM_32BIT_HACK;
7083 break;
7084 }
7085
7086 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7087
7088 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7089 AssertRC(rc);
7090
7091 if (fErrorCodeValid)
7092 TRPMSetErrorCode(pVCpu, uErrorCode);
7093
7094 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7095 && uVector == X86_XCPT_PF)
7096 {
7097 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7098 }
7099 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7100 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7101 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7102 {
7103 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7104 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
7105 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7106 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7107 }
7108
7109 /* Clear the events from the VMCS. */
7110 VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7111
7112 /* We're now done converting the pending event. */
7113 pVCpu->hm.s.Event.fPending = false;
7114}
7115
7116
7117/**
7118 * Does the necessary state syncing before returning to ring-3 for any reason
7119 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7120 *
7121 * @returns VBox status code.
7122 * @param pVCpu The cross context virtual CPU structure.
7123 * @param fImportState Whether to import the guest state from the VMCS back
7124 * to the guest-CPU context.
7125 *
7126 * @remarks No-long-jmp zone!!!
7127 */
7128static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
7129{
7130 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7131 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7132
7133 RTCPUID idCpu = RTMpCpuId();
7134 Log4Func(("HostCpuId=%u\n", idCpu));
7135
7136 /*
7137 * !!! IMPORTANT !!!
7138 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7139 */
7140
7141 /* Save the guest state if necessary. */
7142 if (fImportState)
7143 {
7144 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
7145 AssertRCReturn(rc, rc);
7146 }
7147
7148 /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
7149 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7150 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
7151
7152 /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
7153#ifdef VBOX_STRICT
7154 if (CPUMIsHyperDebugStateActive(pVCpu))
7155 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
7156#endif
7157 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7158 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7159 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7160
7161#if HC_ARCH_BITS == 64
7162 /* Restore host-state bits that VT-x only restores partially. */
7163 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7164 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7165 {
7166 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7167 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7168 }
7169 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7170#endif
7171
7172 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7173 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7174 {
7175 /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
7176 if (!fImportState)
7177 {
7178 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS);
7179 AssertRCReturn(rc, rc);
7180 }
7181 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7182 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7183 }
7184 else
7185 pVCpu->hm.s.vmx.fLazyMsrs = 0;
7186
7187 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7188 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7189
7190 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7191 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
7192 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
7193 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
7194 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
7195 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7196 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7197 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7198 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7199
7200 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7201
7202 /** @todo This partially defeats the purpose of having preemption hooks.
7203 * The problem is, deregistering the hooks should be moved to a place that
7204 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7205 * context.
7206 */
7207 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7208 {
7209 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7210 AssertRCReturn(rc, rc);
7211
7212 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7213 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7214 }
7215 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7216 NOREF(idCpu);
7217
7218 return VINF_SUCCESS;
7219}
7220
7221
7222/**
7223 * Leaves the VT-x session.
7224 *
7225 * @returns VBox status code.
7226 * @param pVCpu The cross context virtual CPU structure.
7227 *
7228 * @remarks No-long-jmp zone!!!
7229 */
7230static int hmR0VmxLeaveSession(PVMCPU pVCpu)
7231{
7232 HM_DISABLE_PREEMPT(pVCpu);
7233 HMVMX_ASSERT_CPU_SAFE(pVCpu);
7234 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7235 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7236
7237 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7238 and done this from the VMXR0ThreadCtxCallback(). */
7239 if (!pVCpu->hm.s.fLeaveDone)
7240 {
7241 int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
7242 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7243 pVCpu->hm.s.fLeaveDone = true;
7244 }
7245 Assert(!pVCpu->cpum.GstCtx.fExtrn);
7246
7247 /*
7248 * !!! IMPORTANT !!!
7249 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7250 */
7251
7252 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7253 /** @todo Deregistering here means we need to VMCLEAR always
7254 * (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
7255 * for calling VMMR0ThreadCtxHookDisable here! */
7256 VMMR0ThreadCtxHookDisable(pVCpu);
7257
7258 /* Leave HM context. This takes care of local init (term). */
7259 int rc = HMR0LeaveCpu(pVCpu);
7260
7261 HM_RESTORE_PREEMPT();
7262 return rc;
7263}
7264
7265
7266/**
7267 * Does the necessary state syncing before doing a longjmp to ring-3.
7268 *
7269 * @returns VBox status code.
7270 * @param pVCpu The cross context virtual CPU structure.
7271 *
7272 * @remarks No-long-jmp zone!!!
7273 */
7274DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
7275{
7276 return hmR0VmxLeaveSession(pVCpu);
7277}
7278
7279
7280/**
7281 * Take necessary actions before going back to ring-3.
7282 *
7283 * An action requires us to go back to ring-3. This function does the necessary
7284 * steps before we can safely return to ring-3. This is not the same as longjmps
7285 * to ring-3, this is voluntary and prepares the guest so it may continue
7286 * executing outside HM (recompiler/IEM).
7287 *
7288 * @returns VBox status code.
7289 * @param pVCpu The cross context virtual CPU structure.
7290 * @param rcExit The reason for exiting to ring-3. Can be
7291 * VINF_VMM_UNKNOWN_RING3_CALL.
7292 */
7293static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
7294{
7295 Assert(pVCpu);
7296 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7297
7298 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7299 {
7300 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7301 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7302 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7303 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7304 }
7305
7306 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7307 VMMRZCallRing3Disable(pVCpu);
7308 Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
7309
7310 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7311 if (pVCpu->hm.s.Event.fPending)
7312 {
7313 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7314 Assert(!pVCpu->hm.s.Event.fPending);
7315 }
7316
7317 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7318 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7319
7320 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7321 and if we're injecting an event we should have a TRPM trap pending. */
7322 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7323#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
7324 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7325#endif
7326
7327 /* Save guest state and restore host state bits. */
7328 int rc = hmR0VmxLeaveSession(pVCpu);
7329 AssertRCReturn(rc, rc);
7330 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7331 /* Thread-context hooks are unregistered at this point!!! */
7332
7333 /* Sync recompiler state. */
7334 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7335 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7336 | CPUM_CHANGED_LDTR
7337 | CPUM_CHANGED_GDTR
7338 | CPUM_CHANGED_IDTR
7339 | CPUM_CHANGED_TR
7340 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7341 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
7342 && CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
7343 {
7344 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7345 }
7346
7347 Assert(!pVCpu->hm.s.fClearTrapFlag);
7348
7349 /* Update the exit-to-ring 3 reason. */
7350 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
7351
7352 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7353 if (rcExit != VINF_EM_RAW_INTERRUPT)
7354 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7355
7356 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7357
7358 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7359 VMMRZCallRing3RemoveNotification(pVCpu);
7360 VMMRZCallRing3Enable(pVCpu);
7361
7362 return rc;
7363}
7364
7365
7366/**
7367 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7368 * longjump to ring-3 and possibly get preempted.
7369 *
7370 * @returns VBox status code.
7371 * @param pVCpu The cross context virtual CPU structure.
7372 * @param enmOperation The operation causing the ring-3 longjump.
7373 * @param pvUser User argument, currently unused, NULL.
7374 */
7375static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7376{
7377 RT_NOREF(pvUser);
7378 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7379 {
7380 /*
7381 * !!! IMPORTANT !!!
7382 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7383 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7384 */
7385 VMMRZCallRing3RemoveNotification(pVCpu);
7386 VMMRZCallRing3Disable(pVCpu);
7387 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7388 RTThreadPreemptDisable(&PreemptState);
7389
7390 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
7391 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7392 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7393
7394#if HC_ARCH_BITS == 64
7395 /* Restore host-state bits that VT-x only restores partially. */
7396 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7397 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7398 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7399 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7400#endif
7401
7402 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7403 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7404 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7405
7406 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7407 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7408 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7409 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7410 {
7411 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7412 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7413 }
7414
7415 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7416 VMMR0ThreadCtxHookDisable(pVCpu);
7417 HMR0LeaveCpu(pVCpu);
7418 RTThreadPreemptRestore(&PreemptState);
7419 return VINF_SUCCESS;
7420 }
7421
7422 Assert(pVCpu);
7423 Assert(pvUser);
7424 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7425 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7426
7427 VMMRZCallRing3Disable(pVCpu);
7428 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7429
7430 Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7431
7432 int rc = hmR0VmxLongJmpToRing3(pVCpu);
7433 AssertRCReturn(rc, rc);
7434
7435 VMMRZCallRing3Enable(pVCpu);
7436 return VINF_SUCCESS;
7437}
7438
7439
7440/**
7441 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7442 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7443 *
7444 * @param pVCpu The cross context virtual CPU structure.
7445 */
7446DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7447{
7448 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7449 {
7450 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7451 {
7452 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_PROC_CTLS_INT_WINDOW_EXIT;
7453 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7454 AssertRC(rc);
7455 Log4Func(("Setup interrupt-window exiting\n"));
7456 }
7457 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7458}
7459
7460
7461/**
7462 * Clears the interrupt-window exiting control in the VMCS.
7463 *
7464 * @param pVCpu The cross context virtual CPU structure.
7465 */
7466DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7467{
7468 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT);
7469 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
7470 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7471 AssertRC(rc);
7472 Log4Func(("Cleared interrupt-window exiting\n"));
7473}
7474
7475
7476/**
7477 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7478 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7479 *
7480 * @param pVCpu The cross context virtual CPU structure.
7481 */
7482DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7483{
7484 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7485 {
7486 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7487 {
7488 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7489 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7490 AssertRC(rc);
7491 Log4Func(("Setup NMI-window exiting\n"));
7492 }
7493 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7494}
7495
7496
7497/**
7498 * Clears the NMI-window exiting control in the VMCS.
7499 *
7500 * @param pVCpu The cross context virtual CPU structure.
7501 */
7502DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7503{
7504 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT);
7505 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7506 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7507 AssertRC(rc);
7508 Log4Func(("Cleared NMI-window exiting\n"));
7509}
7510
7511
7512/**
7513 * Evaluates the event to be delivered to the guest and sets it as the pending
7514 * event.
7515 *
7516 * @returns The VT-x guest-interruptibility state.
7517 * @param pVCpu The cross context virtual CPU structure.
7518 */
7519static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu)
7520{
7521 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7522 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7523 uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu);
7524 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7525 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7526 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7527
7528 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7529 Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7530 Assert(!fBlockSti || pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7531 Assert(!TRPMHasTrap(pVCpu));
7532
7533 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7534 APICUpdatePendingInterrupts(pVCpu);
7535
7536 /*
7537 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7538 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7539 */
7540 /** @todo SMI. SMIs take priority over NMIs. */
7541 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7542 {
7543 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7544 if ( !pVCpu->hm.s.Event.fPending
7545 && !fBlockNmi
7546 && !fBlockSti
7547 && !fBlockMovSS)
7548 {
7549 Log4Func(("Pending NMI\n"));
7550 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INT_INFO_VALID;
7551 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7552
7553 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7554 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7555 }
7556 else
7557 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7558 }
7559 /*
7560 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7561 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7562 */
7563 else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)
7564 && !pVCpu->hm.s.fSingleInstruction)
7565 {
7566 Assert(!DBGFIsStepping(pVCpu));
7567 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7568 AssertRCReturn(rc, 0);
7569 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7570 if ( !pVCpu->hm.s.Event.fPending
7571 && !fBlockInt
7572 && !fBlockSti
7573 && !fBlockMovSS)
7574 {
7575 uint8_t u8Interrupt;
7576 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7577 if (RT_SUCCESS(rc))
7578 {
7579 Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7580 uint32_t u32IntInfo = u8Interrupt
7581 | VMX_EXIT_INT_INFO_VALID
7582 | (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7583
7584 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7585 }
7586 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7587 {
7588 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
7589 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7590 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7591
7592 /*
7593 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7594 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7595 * need to re-set this force-flag here.
7596 */
7597 }
7598 else
7599 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7600 }
7601 else
7602 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7603 }
7604
7605 return fIntrState;
7606}
7607
7608
7609/**
7610 * Injects any pending events into the guest if the guest is in a state to
7611 * receive them.
7612 *
7613 * @returns Strict VBox status code (i.e. informational status codes too).
7614 * @param pVCpu The cross context virtual CPU structure.
7615 * @param fIntrState The VT-x guest-interruptibility state.
7616 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7617 * return VINF_EM_DBG_STEPPED if the event was
7618 * dispatched directly.
7619 */
7620static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, uint32_t fIntrState, bool fStepping)
7621{
7622 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7623 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7624
7625 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7626 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7627
7628 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7629 Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7630 Assert(!fBlockSti || pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7631 Assert(!TRPMHasTrap(pVCpu));
7632
7633 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7634 if (pVCpu->hm.s.Event.fPending)
7635 {
7636 /*
7637 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7638 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7639 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7640 *
7641 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7642 */
7643 uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7644#ifdef VBOX_STRICT
7645 if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
7646 {
7647 bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
7648 Assert(!fBlockInt);
7649 Assert(!fBlockSti);
7650 Assert(!fBlockMovSS);
7651 }
7652 else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
7653 {
7654 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7655 Assert(!fBlockSti);
7656 Assert(!fBlockMovSS);
7657 Assert(!fBlockNmi);
7658 }
7659#endif
7660 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7661 uIntType));
7662
7663 /*
7664 * Inject the event and get any changes to the guest-interruptibility state.
7665 *
7666 * The guest-interruptibility state may need to be updated if we inject the event
7667 * into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
7668 */
7669 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7670 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7671 &fIntrState);
7672 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7673
7674 if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
7675 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7676 else
7677 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7678 }
7679
7680 /*
7681 * Update the guest-interruptibility state.
7682 *
7683 * This is required for the real-on-v86 software interrupt injection case above, as well as
7684 * updates to the guest state from ring-3 or IEM/REM.
7685 */
7686 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
7687 AssertRCReturn(rc, rc);
7688
7689 /*
7690 * There's no need to clear the VM-entry interruption-information field here if we're not
7691 * injecting anything. VT-x clears the valid bit on every VM-exit.
7692 *
7693 * See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7694 */
7695
7696 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7697 NOREF(fBlockMovSS); NOREF(fBlockSti);
7698 return rcStrict;
7699}
7700
7701
7702/**
7703 * Injects a double-fault (\#DF) exception into the VM.
7704 *
7705 * @returns Strict VBox status code (i.e. informational status codes too).
7706 * @param pVCpu The cross context virtual CPU structure.
7707 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7708 * and should return VINF_EM_DBG_STEPPED if the event
7709 * is injected directly (register modified by us, not
7710 * by hardware on VM-entry).
7711 * @param pfIntrState Pointer to the current guest interruptibility-state.
7712 * This interruptibility-state will be updated if
7713 * necessary. This cannot not be NULL.
7714 */
7715DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, bool fStepping, uint32_t *pfIntrState)
7716{
7717 uint32_t const u32IntInfo = X86_XCPT_DF | VMX_EXIT_INT_INFO_VALID
7718 | (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT)
7719 | VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7720 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */, fStepping,
7721 pfIntrState);
7722}
7723
7724
7725/**
7726 * Injects a general-protection (\#GP) fault into the VM.
7727 *
7728 * @returns Strict VBox status code (i.e. informational status codes too).
7729 * @param pVCpu The cross context virtual CPU structure.
7730 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7731 * mode, i.e. in real-mode it's not valid).
7732 * @param u32ErrorCode The error code associated with the \#GP.
7733 * @param fStepping Whether we're running in
7734 * hmR0VmxRunGuestCodeStep() and should return
7735 * VINF_EM_DBG_STEPPED if the event is injected
7736 * directly (register modified by us, not by
7737 * hardware on VM-entry).
7738 * @param pfIntrState Pointer to the current guest interruptibility-state.
7739 * This interruptibility-state will be updated if
7740 * necessary. This cannot not be NULL.
7741 */
7742DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, bool fErrorCodeValid, uint32_t u32ErrorCode, bool fStepping,
7743 uint32_t *pfIntrState)
7744{
7745 uint32_t const u32IntInfo = X86_XCPT_GP | VMX_EXIT_INT_INFO_VALID
7746 | (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT)
7747 | (fErrorCodeValid ? VMX_EXIT_INT_INFO_ERROR_CODE_VALID : 0);
7748 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */, fStepping,
7749 pfIntrState);
7750}
7751
7752
7753/**
7754 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7755 * stack.
7756 *
7757 * @returns Strict VBox status code (i.e. informational status codes too).
7758 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7759 * @param pVCpu The cross context virtual CPU structure.
7760 * @param uValue The value to push to the guest stack.
7761 */
7762static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
7763{
7764 /*
7765 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7766 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7767 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7768 */
7769 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7770 if (pCtx->sp == 1)
7771 return VINF_EM_RESET;
7772 pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7773 int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
7774 AssertRC(rc);
7775 return rc;
7776}
7777
7778
7779/**
7780 * Injects an event into the guest upon VM-entry by updating the relevant fields
7781 * in the VM-entry area in the VMCS.
7782 *
7783 * @returns Strict VBox status code (i.e. informational status codes too).
7784 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7785 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7786 *
7787 * @param pVCpu The cross context virtual CPU structure.
7788 * @param u64IntInfo The VM-entry interruption-information field.
7789 * @param cbInstr The VM-entry instruction length in bytes (for
7790 * software interrupts, exceptions and privileged
7791 * software exceptions).
7792 * @param u32ErrCode The VM-entry exception error code.
7793 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7794 * @param pfIntrState Pointer to the current guest interruptibility-state.
7795 * This interruptibility-state will be updated if
7796 * necessary. This cannot not be NULL.
7797 * @param fStepping Whether we're running in
7798 * hmR0VmxRunGuestCodeStep() and should return
7799 * VINF_EM_DBG_STEPPED if the event is injected
7800 * directly (register modified by us, not by
7801 * hardware on VM-entry).
7802 */
7803static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7804 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7805{
7806 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7807 AssertMsg(!RT_HI_U32(u64IntInfo), ("%#RX64\n", u64IntInfo));
7808 Assert(pfIntrState);
7809
7810 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7811 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7812 uint32_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
7813 uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
7814
7815#ifdef VBOX_STRICT
7816 /*
7817 * Validate the error-code-valid bit for hardware exceptions.
7818 * No error codes for exceptions in real-mode.
7819 *
7820 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7821 */
7822 if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
7823 && !CPUMIsGuestInRealModeEx(pCtx))
7824 {
7825 switch (uVector)
7826 {
7827 case X86_XCPT_PF:
7828 case X86_XCPT_DF:
7829 case X86_XCPT_TS:
7830 case X86_XCPT_NP:
7831 case X86_XCPT_SS:
7832 case X86_XCPT_GP:
7833 case X86_XCPT_AC:
7834 AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
7835 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7836 RT_FALL_THRU();
7837 default:
7838 break;
7839 }
7840 }
7841#endif
7842
7843 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7844 Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
7845 || !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
7846
7847 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7848
7849 /*
7850 * Hardware interrupts & exceptions cannot be delivered through the software interrupt
7851 * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7852 * interrupt handler in the (real-mode) guest.
7853 *
7854 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7855 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7856 */
7857 if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
7858 {
7859 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7860 {
7861 /*
7862 * For unrestricted execution enabled CPUs running real-mode guests, we must not
7863 * set the deliver-error-code bit.
7864 *
7865 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7866 */
7867 u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
7868 }
7869 else
7870 {
7871 PVM pVM = pVCpu->CTX_SUFF(pVM);
7872 Assert(PDMVmmDevHeapIsEnabled(pVM));
7873 Assert(pVM->hm.s.vmx.pRealModeTSS);
7874
7875 /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7876 int rc2 = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_TABLE_MASK | CPUMCTX_EXTRN_RIP
7877 | CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_RFLAGS);
7878 AssertRCReturn(rc2, rc2);
7879
7880 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7881 size_t const cbIdtEntry = sizeof(X86IDTR16);
7882 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
7883 {
7884 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7885 if (uVector == X86_XCPT_DF)
7886 return VINF_EM_RESET;
7887
7888 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7889 if (uVector == X86_XCPT_GP)
7890 return hmR0VmxInjectXcptDF(pVCpu, fStepping, pfIntrState);
7891
7892 /*
7893 * If we're injecting an event with no valid IDT entry, inject a #GP.
7894 * No error codes for exceptions in real-mode.
7895 *
7896 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7897 */
7898 return hmR0VmxInjectXcptGP(pVCpu, false /* fErrCodeValid */, 0 /* u32ErrCode */, fStepping, pfIntrState);
7899 }
7900
7901 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7902 uint16_t uGuestIp = pCtx->ip;
7903 if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
7904 {
7905 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7906 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7907 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7908 }
7909 else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
7910 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7911
7912 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7913 X86IDTR16 IdtEntry;
7914 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
7915 rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7916 AssertRCReturn(rc2, rc2);
7917
7918 /* Construct the stack frame for the interrupt/exception handler. */
7919 VBOXSTRICTRC rcStrict;
7920 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
7921 if (rcStrict == VINF_SUCCESS)
7922 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
7923 if (rcStrict == VINF_SUCCESS)
7924 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
7925
7926 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7927 if (rcStrict == VINF_SUCCESS)
7928 {
7929 pCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7930 pCtx->rip = IdtEntry.offSel;
7931 pCtx->cs.Sel = IdtEntry.uSel;
7932 pCtx->cs.ValidSel = IdtEntry.uSel;
7933 pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7934 if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
7935 && uVector == X86_XCPT_PF)
7936 pCtx->cr2 = GCPtrFaultAddress;
7937
7938 /* If any other guest-state bits are changed here, make sure to update
7939 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7940 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS | HM_CHANGED_GUEST_CR2
7941 | HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
7942 | HM_CHANGED_GUEST_RSP);
7943
7944 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7945 if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
7946 {
7947 Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
7948 && uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
7949 Log4Func(("Clearing inhibition due to STI\n"));
7950 *pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
7951 }
7952 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7953 u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
7954
7955 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7956 it, if we are returning to ring-3 before executing guest code. */
7957 pVCpu->hm.s.Event.fPending = false;
7958
7959 /* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
7960 if (fStepping)
7961 rcStrict = VINF_EM_DBG_STEPPED;
7962 }
7963 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7964 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7965 return rcStrict;
7966 }
7967 }
7968
7969 /* Validate. */
7970 Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7971 Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
7972
7973 /* Inject. */
7974 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7975 if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
7976 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7977 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7978 AssertRCReturn(rc, rc);
7979
7980 /* Update CR2. */
7981 if ( VMX_ENTRY_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
7982 && uVector == X86_XCPT_PF)
7983 pCtx->cr2 = GCPtrFaultAddress;
7984
7985 Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
7986
7987 return VINF_SUCCESS;
7988}
7989
7990
7991/**
7992 * Clears the interrupt-window exiting control in the VMCS and if necessary
7993 * clears the current event in the VMCS as well.
7994 *
7995 * @returns VBox status code.
7996 * @param pVCpu The cross context virtual CPU structure.
7997 *
7998 * @remarks Use this function only to clear events that have not yet been
7999 * delivered to the guest but are injected in the VMCS!
8000 * @remarks No-long-jump zone!!!
8001 */
8002static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8003{
8004 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8005 {
8006 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8007 Log4Func(("Cleared interrupt window\n"));
8008 }
8009
8010 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8011 {
8012 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8013 Log4Func(("Cleared NMI window\n"));
8014 }
8015}
8016
8017
8018/**
8019 * Enters the VT-x session.
8020 *
8021 * @returns VBox status code.
8022 * @param pVCpu The cross context virtual CPU structure.
8023 * @param pHostCpu Pointer to the global CPU info struct.
8024 */
8025VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu, PHMGLOBALCPUINFO pHostCpu)
8026{
8027 AssertPtr(pVCpu);
8028 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8029 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8030 RT_NOREF(pHostCpu);
8031
8032 LogFlowFunc(("pVCpu=%p\n", pVCpu));
8033 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8034 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8035
8036#ifdef VBOX_STRICT
8037 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8038 RTCCUINTREG uHostCR4 = ASMGetCR4();
8039 if (!(uHostCR4 & X86_CR4_VMXE))
8040 {
8041 LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8042 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8043 }
8044#endif
8045
8046 /*
8047 * Load the VCPU's VMCS as the current (and active) one.
8048 */
8049 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8050 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8051 if (RT_FAILURE(rc))
8052 return rc;
8053
8054 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8055 pVCpu->hm.s.fLeaveDone = false;
8056 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8057
8058 return VINF_SUCCESS;
8059}
8060
8061
8062/**
8063 * The thread-context callback (only on platforms which support it).
8064 *
8065 * @param enmEvent The thread-context event.
8066 * @param pVCpu The cross context virtual CPU structure.
8067 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8068 * @thread EMT(pVCpu)
8069 */
8070VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8071{
8072 NOREF(fGlobalInit);
8073
8074 switch (enmEvent)
8075 {
8076 case RTTHREADCTXEVENT_OUT:
8077 {
8078 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8079 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8080 VMCPU_ASSERT_EMT(pVCpu);
8081
8082 /* No longjmps (logger flushes, locks) in this fragile context. */
8083 VMMRZCallRing3Disable(pVCpu);
8084 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8085
8086 /*
8087 * Restore host-state (FPU, debug etc.)
8088 */
8089 if (!pVCpu->hm.s.fLeaveDone)
8090 {
8091 /*
8092 * Do -not- import the guest-state here as we might already be in the middle of importing
8093 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
8094 */
8095 hmR0VmxLeave(pVCpu, false /* fImportState */);
8096 pVCpu->hm.s.fLeaveDone = true;
8097 }
8098
8099 /* Leave HM context, takes care of local init (term). */
8100 int rc = HMR0LeaveCpu(pVCpu);
8101 AssertRC(rc); NOREF(rc);
8102
8103 /* Restore longjmp state. */
8104 VMMRZCallRing3Enable(pVCpu);
8105 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8106 break;
8107 }
8108
8109 case RTTHREADCTXEVENT_IN:
8110 {
8111 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8112 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8113 VMCPU_ASSERT_EMT(pVCpu);
8114
8115 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8116 VMMRZCallRing3Disable(pVCpu);
8117 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8118
8119 /* Initialize the bare minimum state required for HM. This takes care of
8120 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8121 int rc = hmR0EnterCpu(pVCpu);
8122 AssertRC(rc);
8123 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8124 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8125
8126 /* Load the active VMCS as the current one. */
8127 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8128 {
8129 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8130 AssertRC(rc); NOREF(rc);
8131 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8132 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8133 }
8134 pVCpu->hm.s.fLeaveDone = false;
8135
8136 /* Restore longjmp state. */
8137 VMMRZCallRing3Enable(pVCpu);
8138 break;
8139 }
8140
8141 default:
8142 break;
8143 }
8144}
8145
8146
8147/**
8148 * Exports the host state into the VMCS host-state area.
8149 * Sets up the VM-exit MSR-load area.
8150 *
8151 * The CPU state will be loaded from these fields on every successful VM-exit.
8152 *
8153 * @returns VBox status code.
8154 * @param pVCpu The cross context virtual CPU structure.
8155 *
8156 * @remarks No-long-jump zone!!!
8157 */
8158static int hmR0VmxExportHostState(PVMCPU pVCpu)
8159{
8160 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8161
8162 int rc = VINF_SUCCESS;
8163 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8164 {
8165 rc = hmR0VmxExportHostControlRegs();
8166 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8167
8168 rc = hmR0VmxExportHostSegmentRegs(pVCpu);
8169 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8170
8171 rc = hmR0VmxExportHostMsrs(pVCpu);
8172 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8173
8174 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
8175 }
8176 return rc;
8177}
8178
8179
8180/**
8181 * Saves the host state in the VMCS host-state.
8182 *
8183 * @returns VBox status code.
8184 * @param pVCpu The cross context virtual CPU structure.
8185 *
8186 * @remarks No-long-jump zone!!!
8187 */
8188VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
8189{
8190 AssertPtr(pVCpu);
8191 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8192
8193 /*
8194 * Export the host state here while entering HM context.
8195 * When thread-context hooks are used, we might get preempted and have to re-save the host
8196 * state but most of the time we won't be, so do it here before we disable interrupts.
8197 */
8198 return hmR0VmxExportHostState(pVCpu);
8199}
8200
8201
8202/**
8203 * Exports the guest state into the VMCS guest-state area.
8204 *
8205 * The will typically be done before VM-entry when the guest-CPU state and the
8206 * VMCS state may potentially be out of sync.
8207 *
8208 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8209 * VM-entry controls.
8210 * Sets up the appropriate VMX non-root function to execute guest code based on
8211 * the guest CPU mode.
8212 *
8213 * @returns VBox strict status code.
8214 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8215 * without unrestricted guest access and the VMMDev is not presently
8216 * mapped (e.g. EFI32).
8217 *
8218 * @param pVCpu The cross context virtual CPU structure.
8219 *
8220 * @remarks No-long-jump zone!!!
8221 */
8222static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu)
8223{
8224 AssertPtr(pVCpu);
8225 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8226
8227 LogFlowFunc(("pVCpu=%p\n", pVCpu));
8228
8229 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
8230
8231 /* Determine real-on-v86 mode. */
8232 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8233 if ( !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
8234 && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
8235 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8236
8237 /*
8238 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8239 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8240 */
8241 int rc = hmR0VmxSelectVMRunHandler(pVCpu);
8242 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8243
8244 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8245 rc = hmR0VmxExportGuestEntryCtls(pVCpu);
8246 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8247
8248 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8249 rc = hmR0VmxExportGuestExitCtls(pVCpu);
8250 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8251
8252 rc = hmR0VmxExportGuestCR0(pVCpu);
8253 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8254
8255 VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu);
8256 if (rcStrict == VINF_SUCCESS)
8257 { /* likely */ }
8258 else
8259 {
8260 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8261 return rcStrict;
8262 }
8263
8264 rc = hmR0VmxExportGuestSegmentRegs(pVCpu);
8265 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8266
8267 /* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8268 may alter controls if we determine we don't have to swap EFER after all. */
8269 rc = hmR0VmxExportGuestMsrs(pVCpu);
8270 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8271
8272 rc = hmR0VmxExportGuestApicTpr(pVCpu);
8273 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8274
8275 rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8276 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8277
8278 rc = hmR0VmxExportGuestRip(pVCpu);
8279 rc |= hmR0VmxExportGuestRsp(pVCpu);
8280 rc |= hmR0VmxExportGuestRflags(pVCpu);
8281 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8282
8283 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8284 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8285 | HM_CHANGED_GUEST_CR2
8286 | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8287 | HM_CHANGED_GUEST_X87
8288 | HM_CHANGED_GUEST_SSE_AVX
8289 | HM_CHANGED_GUEST_OTHER_XSAVE
8290 | HM_CHANGED_GUEST_XCRx
8291 | HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8292 | HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8293 | HM_CHANGED_GUEST_TSC_AUX
8294 | HM_CHANGED_GUEST_OTHER_MSRS
8295 | HM_CHANGED_GUEST_HWVIRT
8296 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8297
8298 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8299 return rc;
8300}
8301
8302
8303/**
8304 * Exports the state shared between the host and guest into the VMCS.
8305 *
8306 * @param pVCpu The cross context virtual CPU structure.
8307 *
8308 * @remarks No-long-jump zone!!!
8309 */
8310static void hmR0VmxExportSharedState(PVMCPU pVCpu)
8311{
8312 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8313 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8314
8315 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8316 {
8317 int rc = hmR0VmxExportSharedDebugState(pVCpu);
8318 AssertRC(rc);
8319 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8320
8321 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8322 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8323 {
8324 rc = hmR0VmxExportGuestRflags(pVCpu);
8325 AssertRC(rc);
8326 }
8327 }
8328
8329 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8330 {
8331 hmR0VmxLazyLoadGuestMsrs(pVCpu);
8332 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8333 }
8334
8335 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8336 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8337}
8338
8339
8340/**
8341 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8342 *
8343 * @returns Strict VBox status code (i.e. informational status codes too).
8344 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8345 * without unrestricted guest access and the VMMDev is not presently
8346 * mapped (e.g. EFI32).
8347 *
8348 * @param pVCpu The cross context virtual CPU structure.
8349 *
8350 * @remarks No-long-jump zone!!!
8351 */
8352static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu)
8353{
8354 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8355 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8356 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8357
8358#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8359 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8360#endif
8361
8362 /*
8363 * For many exits it's only RIP that changes and hence try to export it first
8364 * without going through a lot of change flag checks.
8365 */
8366 VBOXSTRICTRC rcStrict;
8367 uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8368 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8369 if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8370 {
8371 rcStrict = hmR0VmxExportGuestRip(pVCpu);
8372 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8373 { /* likely */}
8374 else
8375 AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8376 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8377 }
8378 else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8379 {
8380 rcStrict = hmR0VmxExportGuestState(pVCpu);
8381 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8382 { /* likely */}
8383 else
8384 {
8385 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8386 VBOXSTRICTRC_VAL(rcStrict)));
8387 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8388 return rcStrict;
8389 }
8390 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8391 }
8392 else
8393 rcStrict = VINF_SUCCESS;
8394
8395#ifdef VBOX_STRICT
8396 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8397 fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8398 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8399 AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8400 ("fCtxChanged=%#RX64\n", fCtxChanged));
8401#endif
8402 return rcStrict;
8403}
8404
8405
8406/**
8407 * Does the preparations before executing guest code in VT-x.
8408 *
8409 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8410 * recompiler/IEM. We must be cautious what we do here regarding committing
8411 * guest-state information into the VMCS assuming we assuredly execute the
8412 * guest in VT-x mode.
8413 *
8414 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8415 * the common-state (TRPM/forceflags), we must undo those changes so that the
8416 * recompiler/IEM can (and should) use them when it resumes guest execution.
8417 * Otherwise such operations must be done when we can no longer exit to ring-3.
8418 *
8419 * @returns Strict VBox status code (i.e. informational status codes too).
8420 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8421 * have been disabled.
8422 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8423 * double-fault into the guest.
8424 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8425 * dispatched directly.
8426 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8427 *
8428 * @param pVCpu The cross context virtual CPU structure.
8429 * @param pVmxTransient Pointer to the VMX transient structure.
8430 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8431 * us ignore some of the reasons for returning to
8432 * ring-3, and return VINF_EM_DBG_STEPPED if event
8433 * dispatching took place.
8434 */
8435static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
8436{
8437 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8438
8439#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_ONLY_IN_IEM
8440 Log2(("hmR0SvmPreRunGuest: Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
8441 return VINF_EM_RESCHEDULE_REM;
8442#endif
8443
8444#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8445 PGMRZDynMapFlushAutoSet(pVCpu);
8446#endif
8447
8448 /* Check force flag actions that might require us to go back to ring-3. */
8449 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
8450 if (rcStrict == VINF_SUCCESS)
8451 { /* FFs doesn't get set all the time. */ }
8452 else
8453 return rcStrict;
8454
8455 /*
8456 * Setup the virtualized-APIC accesses.
8457 *
8458 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8459 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8460 *
8461 * This is the reason we do it here and not in hmR0VmxExportGuestState().
8462 */
8463 PVM pVM = pVCpu->CTX_SUFF(pVM);
8464 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8465 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
8466 && PDMHasApic(pVM))
8467 {
8468 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8469 Assert(u64MsrApicBase);
8470 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8471
8472 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8473
8474 /* Unalias any existing mapping. */
8475 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8476 AssertRCReturn(rc, rc);
8477
8478 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8479 Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8480 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8481 AssertRCReturn(rc, rc);
8482
8483 /* Update the per-VCPU cache of the APIC base MSR. */
8484 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8485 }
8486
8487 if (TRPMHasTrap(pVCpu))
8488 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8489 uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu);
8490
8491 /*
8492 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
8493 * needs to be done with longjmps or interrupts + preemption enabled. Event injection might
8494 * also result in triple-faulting the VM.
8495 */
8496 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, fIntrState, fStepping);
8497 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8498 { /* likely */ }
8499 else
8500 {
8501 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8502 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8503 return rcStrict;
8504 }
8505
8506 /*
8507 * A longjump might result in importing CR3 even for VM-exits that don't necessarily
8508 * import CR3 themselves. We will need to update them here, as even as late as the above
8509 * hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
8510 * the below force flags to be set.
8511 */
8512 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8513 {
8514 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
8515 int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8516 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
8517 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
8518 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8519 }
8520 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8521 {
8522 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8523 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8524 }
8525
8526 /*
8527 * No longjmps to ring-3 from this point on!!!
8528 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8529 * This also disables flushing of the R0-logger instance (if any).
8530 */
8531 VMMRZCallRing3Disable(pVCpu);
8532
8533 /*
8534 * Export the guest state bits.
8535 *
8536 * We cannot perform longjmps while loading the guest state because we do not preserve the
8537 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8538 * CPU migration.
8539 *
8540 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8541 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8542 * Hence, loading of the guest state needs to be done -after- injection of events.
8543 */
8544 rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu);
8545 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8546 { /* likely */ }
8547 else
8548 {
8549 VMMRZCallRing3Enable(pVCpu);
8550 return rcStrict;
8551 }
8552
8553 /*
8554 * We disable interrupts so that we don't miss any interrupts that would flag preemption
8555 * (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
8556 * preemption disabled for a while. Since this is purly to aid the
8557 * RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
8558 * disable interrupt on NT.
8559 *
8560 * We need to check for force-flags that could've possible been altered since we last
8561 * checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
8562 * see @bugref{6398}).
8563 *
8564 * We also check a couple of other force-flags as a last opportunity to get the EMT back
8565 * to ring-3 before executing guest code.
8566 */
8567 pVmxTransient->fEFlags = ASMIntDisableFlags();
8568
8569 if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8570 && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8571 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8572 && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8573 {
8574 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8575 {
8576 pVCpu->hm.s.Event.fPending = false;
8577
8578 /*
8579 * We've injected any pending events. This is really the point of no return (to ring-3).
8580 *
8581 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8582 * returns from this function, so don't enable them here.
8583 */
8584 return VINF_SUCCESS;
8585 }
8586
8587 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
8588 rcStrict = VINF_EM_RAW_INTERRUPT;
8589 }
8590 else
8591 {
8592 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8593 rcStrict = VINF_EM_RAW_TO_R3;
8594 }
8595
8596 ASMSetFlags(pVmxTransient->fEFlags);
8597 VMMRZCallRing3Enable(pVCpu);
8598
8599 return rcStrict;
8600}
8601
8602
8603/**
8604 * Prepares to run guest code in VT-x and we've committed to doing so. This
8605 * means there is no backing out to ring-3 or anywhere else at this
8606 * point.
8607 *
8608 * @param pVCpu The cross context virtual CPU structure.
8609 * @param pVmxTransient Pointer to the VMX transient structure.
8610 *
8611 * @remarks Called with preemption disabled.
8612 * @remarks No-long-jump zone!!!
8613 */
8614static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
8615{
8616 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8617 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8618 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8619
8620 /*
8621 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8622 */
8623 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8624 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8625
8626 PVM pVM = pVCpu->CTX_SUFF(pVM);
8627 if (!CPUMIsGuestFPUStateActive(pVCpu))
8628 {
8629 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8630 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8631 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
8632 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8633 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8634 }
8635
8636 /*
8637 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8638 */
8639 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8640 && pVCpu->hm.s.vmx.cMsrs > 0)
8641 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8642
8643 /*
8644 * Re-save the host state bits as we may've been preempted (only happens when
8645 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8646 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8647 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8648 * See @bugref{8432}.
8649 */
8650 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8651 {
8652 int rc = hmR0VmxExportHostState(pVCpu);
8653 AssertRC(rc);
8654 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8655 }
8656 Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8657
8658 /*
8659 * Export the state shared between host and guest (FPU, debug, lazy MSRs).
8660 */
8661 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8662 hmR0VmxExportSharedState(pVCpu);
8663 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8664
8665 /* Store status of the shared guest-host state at the time of VM-entry. */
8666#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8667 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
8668 {
8669 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8670 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8671 }
8672 else
8673#endif
8674 {
8675 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8676 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8677 }
8678
8679 /*
8680 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8681 */
8682 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
8683 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8684
8685 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8686 RTCPUID idCurrentCpu = pCpu->idCpu;
8687 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8688 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8689 {
8690 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8691 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8692 }
8693
8694 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8695 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8696 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8697 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8698
8699 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8700
8701 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8702 to start executing. */
8703
8704 /*
8705 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8706 */
8707 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
8708 {
8709 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
8710 {
8711 bool fMsrUpdated;
8712 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8713 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8714 &fMsrUpdated);
8715 AssertRC(rc2);
8716 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8717 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8718 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8719 }
8720 else
8721 {
8722 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8723 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8724 }
8725 }
8726
8727 if (pVM->cpum.ro.GuestFeatures.fIbrs)
8728 {
8729 bool fMsrUpdated;
8730 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8731 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8732 &fMsrUpdated);
8733 AssertRC(rc2);
8734 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8735 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8736 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8737 }
8738
8739#ifdef VBOX_STRICT
8740 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8741 hmR0VmxCheckHostEferMsr(pVCpu);
8742 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8743#endif
8744#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8745 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
8746 {
8747 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
8748 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8749 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8750 }
8751#endif
8752}
8753
8754
8755/**
8756 * Performs some essential restoration of state after running guest code in
8757 * VT-x.
8758 *
8759 * @param pVCpu The cross context virtual CPU structure.
8760 * @param pVmxTransient Pointer to the VMX transient structure.
8761 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8762 *
8763 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8764 *
8765 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8766 * unconditionally when it is safe to do so.
8767 */
8768static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8769{
8770 uint64_t const uHostTsc = ASMReadTSC();
8771 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8772
8773 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8774 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8775 pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8776 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8777 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8778 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8779
8780 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
8781 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TscOffset);
8782
8783 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
8784 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8785 Assert(!ASMIntAreEnabled());
8786 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8787
8788#if HC_ARCH_BITS == 64
8789 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8790#endif
8791#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8792 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8793 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8794 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8795#else
8796 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8797#endif
8798#ifdef VBOX_STRICT
8799 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8800#endif
8801 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8802
8803 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8804 uint32_t uExitReason;
8805 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8806 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8807 AssertRC(rc);
8808 pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
8809 pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
8810
8811 if (rcVMRun == VINF_SUCCESS)
8812 {
8813 /*
8814 * Update the VM-exit history array here even if the VM-entry failed due to:
8815 * - Invalid guest state.
8816 * - MSR loading.
8817 * - Machine-check event.
8818 *
8819 * In any of the above cases we will still have a "valid" VM-exit reason
8820 * despite @a fVMEntryFailed being false.
8821 *
8822 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8823 *
8824 * Note! We don't have CS or RIP at this point. Will probably address that later
8825 * by amending the history entry added here.
8826 */
8827 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8828 UINT64_MAX, uHostTsc);
8829
8830 if (!pVmxTransient->fVMEntryFailed)
8831 {
8832 VMMRZCallRing3Enable(pVCpu);
8833
8834 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8835 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8836
8837#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8838 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8839 AssertRC(rc);
8840#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8841 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8842 AssertRC(rc);
8843#else
8844 /*
8845 * Import the guest-interruptibility state always as we need it while evaluating
8846 * injecting events on re-entry.
8847 *
8848 * We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8849 * checking for real-mode while exporting the state because all bits that cause
8850 * mode changes wrt CR0 are intercepted.
8851 */
8852 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8853 AssertRC(rc);
8854#endif
8855
8856 /*
8857 * Sync the TPR shadow with our APIC state.
8858 */
8859 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
8860 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8861 {
8862 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8863 AssertRC(rc);
8864 ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8865 }
8866
8867 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8868 return;
8869 }
8870 }
8871 else
8872 Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8873
8874 VMMRZCallRing3Enable(pVCpu);
8875}
8876
8877
8878/**
8879 * Runs the guest code using VT-x the normal way.
8880 *
8881 * @returns VBox status code.
8882 * @param pVCpu The cross context virtual CPU structure.
8883 *
8884 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8885 */
8886static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu)
8887{
8888 VMXTRANSIENT VmxTransient;
8889 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8890 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8891 uint32_t cLoops = 0;
8892
8893 for (;; cLoops++)
8894 {
8895 Assert(!HMR0SuspendPending());
8896 HMVMX_ASSERT_CPU_SAFE(pVCpu);
8897
8898 /* Preparatory work for running guest code, this may force us to return
8899 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8900 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8901 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
8902 if (rcStrict != VINF_SUCCESS)
8903 break;
8904
8905 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
8906 int rcRun = hmR0VmxRunGuest(pVCpu);
8907
8908 /* Restore any residual host-state and save any bits shared between host
8909 and guest into the guest-CPU state. Re-enables interrupts! */
8910 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8911
8912 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8913 if (RT_SUCCESS(rcRun))
8914 { /* very likely */ }
8915 else
8916 {
8917 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
8918 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
8919 return rcRun;
8920 }
8921
8922 /* Profile the VM-exit. */
8923 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8924 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8925 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8926 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
8927 HMVMX_START_EXIT_DISPATCH_PROF();
8928
8929 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
8930
8931 /* Handle the VM-exit. */
8932#ifdef HMVMX_USE_FUNCTION_TABLE
8933 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
8934#else
8935 rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient, VmxTransient.uExitReason);
8936#endif
8937 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
8938 if (rcStrict == VINF_SUCCESS)
8939 {
8940 if (cLoops <= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
8941 continue; /* likely */
8942 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8943 rcStrict = VINF_EM_RAW_INTERRUPT;
8944 }
8945 break;
8946 }
8947
8948 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8949 return rcStrict;
8950}
8951
8952
8953
8954/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8955 * probes.
8956 *
8957 * The following few functions and associated structure contains the bloat
8958 * necessary for providing detailed debug events and dtrace probes as well as
8959 * reliable host side single stepping. This works on the principle of
8960 * "subclassing" the normal execution loop and workers. We replace the loop
8961 * method completely and override selected helpers to add necessary adjustments
8962 * to their core operation.
8963 *
8964 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8965 * any performance for debug and analysis features.
8966 *
8967 * @{
8968 */
8969
8970/**
8971 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
8972 * the debug run loop.
8973 */
8974typedef struct VMXRUNDBGSTATE
8975{
8976 /** The RIP we started executing at. This is for detecting that we stepped. */
8977 uint64_t uRipStart;
8978 /** The CS we started executing with. */
8979 uint16_t uCsStart;
8980
8981 /** Whether we've actually modified the 1st execution control field. */
8982 bool fModifiedProcCtls : 1;
8983 /** Whether we've actually modified the 2nd execution control field. */
8984 bool fModifiedProcCtls2 : 1;
8985 /** Whether we've actually modified the exception bitmap. */
8986 bool fModifiedXcptBitmap : 1;
8987
8988 /** We desire the modified the CR0 mask to be cleared. */
8989 bool fClearCr0Mask : 1;
8990 /** We desire the modified the CR4 mask to be cleared. */
8991 bool fClearCr4Mask : 1;
8992 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8993 uint32_t fCpe1Extra;
8994 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
8995 uint32_t fCpe1Unwanted;
8996 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
8997 uint32_t fCpe2Extra;
8998 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
8999 uint32_t bmXcptExtra;
9000 /** The sequence number of the Dtrace provider settings the state was
9001 * configured against. */
9002 uint32_t uDtraceSettingsSeqNo;
9003 /** VM-exits to check (one bit per VM-exit). */
9004 uint32_t bmExitsToCheck[3];
9005
9006 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9007 uint32_t fProcCtlsInitial;
9008 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9009 uint32_t fProcCtls2Initial;
9010 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9011 uint32_t bmXcptInitial;
9012} VMXRUNDBGSTATE;
9013AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9014typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9015
9016
9017/**
9018 * Initializes the VMXRUNDBGSTATE structure.
9019 *
9020 * @param pVCpu The cross context virtual CPU structure of the
9021 * calling EMT.
9022 * @param pDbgState The structure to initialize.
9023 */
9024static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9025{
9026 pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
9027 pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
9028
9029 pDbgState->fModifiedProcCtls = false;
9030 pDbgState->fModifiedProcCtls2 = false;
9031 pDbgState->fModifiedXcptBitmap = false;
9032 pDbgState->fClearCr0Mask = false;
9033 pDbgState->fClearCr4Mask = false;
9034 pDbgState->fCpe1Extra = 0;
9035 pDbgState->fCpe1Unwanted = 0;
9036 pDbgState->fCpe2Extra = 0;
9037 pDbgState->bmXcptExtra = 0;
9038 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9039 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9040 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9041}
9042
9043
9044/**
9045 * Updates the VMSC fields with changes requested by @a pDbgState.
9046 *
9047 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9048 * immediately before executing guest code, i.e. when interrupts are disabled.
9049 * We don't check status codes here as we cannot easily assert or return in the
9050 * latter case.
9051 *
9052 * @param pVCpu The cross context virtual CPU structure.
9053 * @param pDbgState The debug state.
9054 */
9055static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9056{
9057 /*
9058 * Ensure desired flags in VMCS control fields are set.
9059 * (Ignoring write failure here, as we're committed and it's just debug extras.)
9060 *
9061 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9062 * there should be no stale data in pCtx at this point.
9063 */
9064 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9065 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9066 {
9067 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
9068 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9069 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9070 Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9071 pDbgState->fModifiedProcCtls = true;
9072 }
9073
9074 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9075 {
9076 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9077 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9078 Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9079 pDbgState->fModifiedProcCtls2 = true;
9080 }
9081
9082 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9083 {
9084 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9085 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9086 Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9087 pDbgState->fModifiedXcptBitmap = true;
9088 }
9089
9090 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32Cr0Mask != 0)
9091 {
9092 pVCpu->hm.s.vmx.u32Cr0Mask = 0;
9093 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9094 Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9095 }
9096
9097 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32Cr4Mask != 0)
9098 {
9099 pVCpu->hm.s.vmx.u32Cr4Mask = 0;
9100 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9101 Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9102 }
9103}
9104
9105
9106/**
9107 * Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
9108 * re-entry next time around.
9109 *
9110 * @returns Strict VBox status code (i.e. informational status codes too).
9111 * @param pVCpu The cross context virtual CPU structure.
9112 * @param pDbgState The debug state.
9113 * @param rcStrict The return code from executing the guest using single
9114 * stepping.
9115 */
9116static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9117{
9118 /*
9119 * Restore VM-exit control settings as we may not reenter this function the
9120 * next time around.
9121 */
9122 /* We reload the initial value, trigger what we can of recalculations the
9123 next time around. From the looks of things, that's all that's required atm. */
9124 if (pDbgState->fModifiedProcCtls)
9125 {
9126 if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9127 pDbgState->fProcCtlsInitial |= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9128 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9129 AssertRCReturn(rc2, rc2);
9130 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9131 }
9132
9133 /* We're currently the only ones messing with this one, so just restore the
9134 cached value and reload the field. */
9135 if ( pDbgState->fModifiedProcCtls2
9136 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9137 {
9138 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9139 AssertRCReturn(rc2, rc2);
9140 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9141 }
9142
9143 /* If we've modified the exception bitmap, we restore it and trigger
9144 reloading and partial recalculation the next time around. */
9145 if (pDbgState->fModifiedXcptBitmap)
9146 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9147
9148 return rcStrict;
9149}
9150
9151
9152/**
9153 * Configures VM-exit controls for current DBGF and DTrace settings.
9154 *
9155 * This updates @a pDbgState and the VMCS execution control fields to reflect
9156 * the necessary VM-exits demanded by DBGF and DTrace.
9157 *
9158 * @param pVCpu The cross context virtual CPU structure.
9159 * @param pDbgState The debug state.
9160 * @param pVmxTransient Pointer to the VMX transient structure. May update
9161 * fUpdateTscOffsettingAndPreemptTimer.
9162 */
9163static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9164{
9165 /*
9166 * Take down the dtrace serial number so we can spot changes.
9167 */
9168 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9169 ASMCompilerBarrier();
9170
9171 /*
9172 * We'll rebuild most of the middle block of data members (holding the
9173 * current settings) as we go along here, so start by clearing it all.
9174 */
9175 pDbgState->bmXcptExtra = 0;
9176 pDbgState->fCpe1Extra = 0;
9177 pDbgState->fCpe1Unwanted = 0;
9178 pDbgState->fCpe2Extra = 0;
9179 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9180 pDbgState->bmExitsToCheck[i] = 0;
9181
9182 /*
9183 * Software interrupts (INT XXh) - no idea how to trigger these...
9184 */
9185 PVM pVM = pVCpu->CTX_SUFF(pVM);
9186 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9187 || VBOXVMM_INT_SOFTWARE_ENABLED())
9188 {
9189 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9190 }
9191
9192 /*
9193 * INT3 breakpoints - triggered by #BP exceptions.
9194 */
9195 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9196 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9197
9198 /*
9199 * Exception bitmap and XCPT events+probes.
9200 */
9201 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9202 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9203 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9204
9205 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9206 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9207 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9208 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9209 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9210 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9211 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9212 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9213 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9214 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9215 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9216 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9217 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9218 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9219 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9220 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9221 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9222 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9223
9224 if (pDbgState->bmXcptExtra)
9225 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9226
9227 /*
9228 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9229 *
9230 * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
9231 * So, when adding/changing/removing please don't forget to update it.
9232 *
9233 * Some of the macros are picking up local variables to save horizontal space,
9234 * (being able to see it in a table is the lesser evil here).
9235 */
9236#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9237 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9238 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9239#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9240 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9241 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9242 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9243 } else do { } while (0)
9244#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9245 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9246 { \
9247 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9248 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9249 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9250 } else do { } while (0)
9251#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9252 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9253 { \
9254 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9255 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9256 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9257 } else do { } while (0)
9258#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9259 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9260 { \
9261 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9262 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9263 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9264 } else do { } while (0)
9265
9266 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9267 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9268 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9269 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9270 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9271
9272 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9273 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9274 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9275 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9276 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
9277 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9278 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9279 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9280 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
9281 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9282 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
9283 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9284 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
9285 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9286 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9287 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9288 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9289 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9290 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9291 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9292 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9293 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9294 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9295 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9296 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9297 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9298 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9299 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9300 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9301 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9302 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9303 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9304 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9305 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9306 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9307 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9308
9309 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9310 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9311 {
9312 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_APIC_TPR);
9313 AssertRC(rc);
9314
9315#if 0 /** @todo fix me */
9316 pDbgState->fClearCr0Mask = true;
9317 pDbgState->fClearCr4Mask = true;
9318#endif
9319 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9320 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_STORE_EXIT | VMX_PROC_CTLS_CR8_STORE_EXIT;
9321 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9322 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_LOAD_EXIT | VMX_PROC_CTLS_CR8_LOAD_EXIT;
9323 pDbgState->fCpe1Unwanted |= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
9324 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9325 require clearing here and in the loop if we start using it. */
9326 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9327 }
9328 else
9329 {
9330 if (pDbgState->fClearCr0Mask)
9331 {
9332 pDbgState->fClearCr0Mask = false;
9333 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9334 }
9335 if (pDbgState->fClearCr4Mask)
9336 {
9337 pDbgState->fClearCr4Mask = false;
9338 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9339 }
9340 }
9341 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9342 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9343
9344 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9345 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9346 {
9347 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9348 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9349 }
9350 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9351 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9352
9353 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
9354 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9355 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
9356 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9357 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
9358 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9359 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
9360 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9361#if 0 /** @todo too slow, fix handler. */
9362 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
9363#endif
9364 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9365
9366 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9367 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9368 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9369 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9370 {
9371 pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
9372 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
9373 }
9374 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9375 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9376 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9377 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9378
9379 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9380 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9381 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9382 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9383 {
9384 pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
9385 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
9386 }
9387 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
9388 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
9389 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
9390 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
9391
9392 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9393 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9394 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
9395 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9396 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9397 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9398 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
9399 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9400 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9401 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9402 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
9403 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9404 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
9405 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9406 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9407 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9408 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
9409 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9410 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9411 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9412 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9413 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9414
9415#undef IS_EITHER_ENABLED
9416#undef SET_ONLY_XBM_IF_EITHER_EN
9417#undef SET_CPE1_XBM_IF_EITHER_EN
9418#undef SET_CPEU_XBM_IF_EITHER_EN
9419#undef SET_CPE2_XBM_IF_EITHER_EN
9420
9421 /*
9422 * Sanitize the control stuff.
9423 */
9424 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
9425 if (pDbgState->fCpe2Extra)
9426 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
9427 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
9428 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0;
9429 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
9430 {
9431 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9432 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9433 }
9434
9435 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9436 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9437 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9438 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9439}
9440
9441
9442/**
9443 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9444 * appropriate.
9445 *
9446 * The caller has checked the VM-exit against the
9447 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9448 * already, so we don't have to do that either.
9449 *
9450 * @returns Strict VBox status code (i.e. informational status codes too).
9451 * @param pVCpu The cross context virtual CPU structure.
9452 * @param pVmxTransient Pointer to the VMX-transient structure.
9453 * @param uExitReason The VM-exit reason.
9454 *
9455 * @remarks The name of this function is displayed by dtrace, so keep it short
9456 * and to the point. No longer than 33 chars long, please.
9457 */
9458static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9459{
9460 /*
9461 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9462 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9463 *
9464 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9465 * does. Must add/change/remove both places. Same ordering, please.
9466 *
9467 * Added/removed events must also be reflected in the next section
9468 * where we dispatch dtrace events.
9469 */
9470 bool fDtrace1 = false;
9471 bool fDtrace2 = false;
9472 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9473 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9474 uint32_t uEventArg = 0;
9475#define SET_EXIT(a_EventSubName) \
9476 do { \
9477 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9478 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9479 } while (0)
9480#define SET_BOTH(a_EventSubName) \
9481 do { \
9482 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9483 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9484 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9485 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9486 } while (0)
9487 switch (uExitReason)
9488 {
9489 case VMX_EXIT_MTF:
9490 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9491
9492 case VMX_EXIT_XCPT_OR_NMI:
9493 {
9494 uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9495 switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
9496 {
9497 case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
9498 case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
9499 case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
9500 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9501 {
9502 if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
9503 {
9504 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9505 uEventArg = pVmxTransient->uExitIntErrorCode;
9506 }
9507 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9508 switch (enmEvent1)
9509 {
9510 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9511 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9512 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9513 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9514 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9515 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9516 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9517 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9518 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9519 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9520 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9521 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9522 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9523 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9524 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9525 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9526 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9527 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9528 default: break;
9529 }
9530 }
9531 else
9532 AssertFailed();
9533 break;
9534
9535 case VMX_EXIT_INT_INFO_TYPE_SW_INT:
9536 uEventArg = idxVector;
9537 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9538 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9539 break;
9540 }
9541 break;
9542 }
9543
9544 case VMX_EXIT_TRIPLE_FAULT:
9545 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9546 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9547 break;
9548 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9549 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9550 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9551 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9552 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9553
9554 /* Instruction specific VM-exits: */
9555 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9556 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9557 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9558 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9559 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9560 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9561 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9562 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9563 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9564 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9565 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9566 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9567 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9568 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9569 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9570 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9571 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9572 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9573 case VMX_EXIT_MOV_CRX:
9574 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9575 if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9576 SET_BOTH(CRX_READ);
9577 else
9578 SET_BOTH(CRX_WRITE);
9579 uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
9580 break;
9581 case VMX_EXIT_MOV_DRX:
9582 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9583 if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
9584 == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9585 SET_BOTH(DRX_READ);
9586 else
9587 SET_BOTH(DRX_WRITE);
9588 uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
9589 break;
9590 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9591 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9592 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9593 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9594 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9595 case VMX_EXIT_GDTR_IDTR_ACCESS:
9596 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9597 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
9598 {
9599 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9600 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9601 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9602 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9603 }
9604 break;
9605
9606 case VMX_EXIT_LDTR_TR_ACCESS:
9607 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9608 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
9609 {
9610 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9611 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9612 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9613 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9614 }
9615 break;
9616
9617 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9618 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9619 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9620 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9621 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9622 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9623 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9624 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9625 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9626 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9627 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9628
9629 /* Events that aren't relevant at this point. */
9630 case VMX_EXIT_EXT_INT:
9631 case VMX_EXIT_INT_WINDOW:
9632 case VMX_EXIT_NMI_WINDOW:
9633 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9634 case VMX_EXIT_PREEMPT_TIMER:
9635 case VMX_EXIT_IO_INSTR:
9636 break;
9637
9638 /* Errors and unexpected events. */
9639 case VMX_EXIT_INIT_SIGNAL:
9640 case VMX_EXIT_SIPI:
9641 case VMX_EXIT_IO_SMI:
9642 case VMX_EXIT_SMI:
9643 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9644 case VMX_EXIT_ERR_MSR_LOAD:
9645 case VMX_EXIT_ERR_MACHINE_CHECK:
9646 break;
9647
9648 default:
9649 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9650 break;
9651 }
9652#undef SET_BOTH
9653#undef SET_EXIT
9654
9655 /*
9656 * Dtrace tracepoints go first. We do them here at once so we don't
9657 * have to copy the guest state saving and stuff a few dozen times.
9658 * Down side is that we've got to repeat the switch, though this time
9659 * we use enmEvent since the probes are a subset of what DBGF does.
9660 */
9661 if (fDtrace1 || fDtrace2)
9662 {
9663 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9664 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9665 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9666 switch (enmEvent1)
9667 {
9668 /** @todo consider which extra parameters would be helpful for each probe. */
9669 case DBGFEVENT_END: break;
9670 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
9671 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
9672 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
9673 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
9674 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
9675 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
9676 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
9677 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
9678 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
9679 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
9680 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
9681 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
9682 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
9683 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
9684 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
9685 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
9686 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
9687 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
9688 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9689 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9690 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
9691 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
9692 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
9693 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
9694 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
9695 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
9696 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
9697 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9698 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9699 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9700 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9701 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9702 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
9703 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9704 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
9705 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
9706 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
9707 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
9708 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
9709 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
9710 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
9711 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
9712 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
9713 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
9714 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
9715 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
9716 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
9717 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
9718 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
9719 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
9720 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
9721 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
9722 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
9723 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
9724 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
9725 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
9726 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
9727 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
9728 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
9729 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
9730 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
9731 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
9732 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
9733 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
9734 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
9735 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
9736 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9737 }
9738 switch (enmEvent2)
9739 {
9740 /** @todo consider which extra parameters would be helpful for each probe. */
9741 case DBGFEVENT_END: break;
9742 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
9743 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9744 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
9745 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
9746 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
9747 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
9748 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
9749 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
9750 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
9751 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9752 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9753 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9754 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9755 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9756 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
9757 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9758 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
9759 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
9760 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
9761 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
9762 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
9763 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
9764 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
9765 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
9766 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
9767 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
9768 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
9769 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
9770 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
9771 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
9772 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
9773 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
9774 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
9775 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
9776 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
9777 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
9778 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
9779 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
9780 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
9781 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
9782 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
9783 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
9784 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
9785 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
9786 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
9787 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
9788 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
9789 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
9790 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
9791 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
9792 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
9793 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
9794 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9795 }
9796 }
9797
9798 /*
9799 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9800 * the DBGF call will do a full check).
9801 *
9802 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9803 * Note! If we have to events, we prioritize the first, i.e. the instruction
9804 * one, in order to avoid event nesting.
9805 */
9806 PVM pVM = pVCpu->CTX_SUFF(pVM);
9807 if ( enmEvent1 != DBGFEVENT_END
9808 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9809 {
9810 HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
9811 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
9812 if (rcStrict != VINF_SUCCESS)
9813 return rcStrict;
9814 }
9815 else if ( enmEvent2 != DBGFEVENT_END
9816 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9817 {
9818 HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
9819 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
9820 if (rcStrict != VINF_SUCCESS)
9821 return rcStrict;
9822 }
9823
9824 return VINF_SUCCESS;
9825}
9826
9827
9828/**
9829 * Single-stepping VM-exit filtering.
9830 *
9831 * This is preprocessing the VM-exits and deciding whether we've gotten far
9832 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9833 * handling is performed.
9834 *
9835 * @returns Strict VBox status code (i.e. informational status codes too).
9836 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9837 * @param pVmxTransient Pointer to the VMX-transient structure.
9838 * @param pDbgState The debug state.
9839 */
9840DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
9841{
9842 /*
9843 * Expensive (saves context) generic dtrace VM-exit probe.
9844 */
9845 uint32_t const uExitReason = pVmxTransient->uExitReason;
9846 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9847 { /* more likely */ }
9848 else
9849 {
9850 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9851 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9852 AssertRC(rc);
9853 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
9854 }
9855
9856 /*
9857 * Check for host NMI, just to get that out of the way.
9858 */
9859 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9860 { /* normally likely */ }
9861 else
9862 {
9863 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9864 AssertRCReturn(rc2, rc2);
9865 uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
9866 if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
9867 return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
9868 }
9869
9870 /*
9871 * Check for single stepping event if we're stepping.
9872 */
9873 if (pVCpu->hm.s.fSingleInstruction)
9874 {
9875 switch (uExitReason)
9876 {
9877 case VMX_EXIT_MTF:
9878 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9879
9880 /* Various events: */
9881 case VMX_EXIT_XCPT_OR_NMI:
9882 case VMX_EXIT_EXT_INT:
9883 case VMX_EXIT_TRIPLE_FAULT:
9884 case VMX_EXIT_INT_WINDOW:
9885 case VMX_EXIT_NMI_WINDOW:
9886 case VMX_EXIT_TASK_SWITCH:
9887 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9888 case VMX_EXIT_APIC_ACCESS:
9889 case VMX_EXIT_EPT_VIOLATION:
9890 case VMX_EXIT_EPT_MISCONFIG:
9891 case VMX_EXIT_PREEMPT_TIMER:
9892
9893 /* Instruction specific VM-exits: */
9894 case VMX_EXIT_CPUID:
9895 case VMX_EXIT_GETSEC:
9896 case VMX_EXIT_HLT:
9897 case VMX_EXIT_INVD:
9898 case VMX_EXIT_INVLPG:
9899 case VMX_EXIT_RDPMC:
9900 case VMX_EXIT_RDTSC:
9901 case VMX_EXIT_RSM:
9902 case VMX_EXIT_VMCALL:
9903 case VMX_EXIT_VMCLEAR:
9904 case VMX_EXIT_VMLAUNCH:
9905 case VMX_EXIT_VMPTRLD:
9906 case VMX_EXIT_VMPTRST:
9907 case VMX_EXIT_VMREAD:
9908 case VMX_EXIT_VMRESUME:
9909 case VMX_EXIT_VMWRITE:
9910 case VMX_EXIT_VMXOFF:
9911 case VMX_EXIT_VMXON:
9912 case VMX_EXIT_MOV_CRX:
9913 case VMX_EXIT_MOV_DRX:
9914 case VMX_EXIT_IO_INSTR:
9915 case VMX_EXIT_RDMSR:
9916 case VMX_EXIT_WRMSR:
9917 case VMX_EXIT_MWAIT:
9918 case VMX_EXIT_MONITOR:
9919 case VMX_EXIT_PAUSE:
9920 case VMX_EXIT_GDTR_IDTR_ACCESS:
9921 case VMX_EXIT_LDTR_TR_ACCESS:
9922 case VMX_EXIT_INVEPT:
9923 case VMX_EXIT_RDTSCP:
9924 case VMX_EXIT_INVVPID:
9925 case VMX_EXIT_WBINVD:
9926 case VMX_EXIT_XSETBV:
9927 case VMX_EXIT_RDRAND:
9928 case VMX_EXIT_INVPCID:
9929 case VMX_EXIT_VMFUNC:
9930 case VMX_EXIT_RDSEED:
9931 case VMX_EXIT_XSAVES:
9932 case VMX_EXIT_XRSTORS:
9933 {
9934 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
9935 AssertRCReturn(rc, rc);
9936 if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
9937 || pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
9938 return VINF_EM_DBG_STEPPED;
9939 break;
9940 }
9941
9942 /* Errors and unexpected events: */
9943 case VMX_EXIT_INIT_SIGNAL:
9944 case VMX_EXIT_SIPI:
9945 case VMX_EXIT_IO_SMI:
9946 case VMX_EXIT_SMI:
9947 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9948 case VMX_EXIT_ERR_MSR_LOAD:
9949 case VMX_EXIT_ERR_MACHINE_CHECK:
9950 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9951 break;
9952
9953 default:
9954 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9955 break;
9956 }
9957 }
9958
9959 /*
9960 * Check for debugger event breakpoints and dtrace probes.
9961 */
9962 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9963 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9964 {
9965 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
9966 if (rcStrict != VINF_SUCCESS)
9967 return rcStrict;
9968 }
9969
9970 /*
9971 * Normal processing.
9972 */
9973#ifdef HMVMX_USE_FUNCTION_TABLE
9974 return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
9975#else
9976 return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
9977#endif
9978}
9979
9980
9981/**
9982 * Single steps guest code using VT-x.
9983 *
9984 * @returns Strict VBox status code (i.e. informational status codes too).
9985 * @param pVCpu The cross context virtual CPU structure.
9986 *
9987 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
9988 */
9989static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu)
9990{
9991 VMXTRANSIENT VmxTransient;
9992 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9993
9994 /* Set HMCPU indicators. */
9995 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
9996 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
9997 pVCpu->hm.s.fDebugWantRdTscExit = false;
9998 pVCpu->hm.s.fUsingDebugLoop = true;
9999
10000 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10001 VMXRUNDBGSTATE DbgState;
10002 hmR0VmxRunDebugStateInit(pVCpu, &DbgState);
10003 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
10004
10005 /*
10006 * The loop.
10007 */
10008 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10009 for (uint32_t cLoops = 0; ; cLoops++)
10010 {
10011 Assert(!HMR0SuspendPending());
10012 HMVMX_ASSERT_CPU_SAFE(pVCpu);
10013 bool fStepping = pVCpu->hm.s.fSingleInstruction;
10014
10015 /*
10016 * Preparatory work for running guest code, this may force us to return
10017 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10018 */
10019 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10020 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10021 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
10022 if (rcStrict != VINF_SUCCESS)
10023 break;
10024
10025 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10026 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10027
10028 /*
10029 * Now we can run the guest code.
10030 */
10031 int rcRun = hmR0VmxRunGuest(pVCpu);
10032
10033 /*
10034 * Restore any residual host-state and save any bits shared between host
10035 * and guest into the guest-CPU state. Re-enables interrupts!
10036 */
10037 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10038
10039 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10040 if (RT_SUCCESS(rcRun))
10041 { /* very likely */ }
10042 else
10043 {
10044 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10045 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10046 return rcRun;
10047 }
10048
10049 /* Profile the VM-exit. */
10050 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10051 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10052 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10053 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10054 HMVMX_START_EXIT_DISPATCH_PROF();
10055
10056 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10057
10058 /*
10059 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10060 */
10061 rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
10062 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10063 if (rcStrict != VINF_SUCCESS)
10064 break;
10065 if (cLoops > pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
10066 {
10067 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10068 rcStrict = VINF_EM_RAW_INTERRUPT;
10069 break;
10070 }
10071
10072 /*
10073 * Stepping: Did the RIP change, if so, consider it a single step.
10074 * Otherwise, make sure one of the TFs gets set.
10075 */
10076 if (fStepping)
10077 {
10078 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
10079 AssertRC(rc);
10080 if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
10081 || pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
10082 {
10083 rcStrict = VINF_EM_DBG_STEPPED;
10084 break;
10085 }
10086 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
10087 }
10088
10089 /*
10090 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10091 */
10092 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10093 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
10094 }
10095
10096 /*
10097 * Clear the X86_EFL_TF if necessary.
10098 */
10099 if (pVCpu->hm.s.fClearTrapFlag)
10100 {
10101 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
10102 AssertRC(rc);
10103 pVCpu->hm.s.fClearTrapFlag = false;
10104 pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
10105 }
10106 /** @todo there seems to be issues with the resume flag when the monitor trap
10107 * flag is pending without being used. Seen early in bios init when
10108 * accessing APIC page in protected mode. */
10109
10110 /*
10111 * Restore VM-exit control settings as we may not reenter this function the
10112 * next time around.
10113 */
10114 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10115
10116 /* Restore HMCPU indicators. */
10117 pVCpu->hm.s.fUsingDebugLoop = false;
10118 pVCpu->hm.s.fDebugWantRdTscExit = false;
10119 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10120
10121 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10122 return rcStrict;
10123}
10124
10125
10126/** @} */
10127
10128
10129/**
10130 * Checks if any expensive dtrace probes are enabled and we should go to the
10131 * debug loop.
10132 *
10133 * @returns true if we should use debug loop, false if not.
10134 */
10135static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10136{
10137 /* It's probably faster to OR the raw 32-bit counter variables together.
10138 Since the variables are in an array and the probes are next to one
10139 another (more or less), we have good locality. So, better read
10140 eight-nine cache lines ever time and only have one conditional, than
10141 128+ conditionals, right? */
10142 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10143 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10144 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10145 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10146 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10147 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10148 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10149 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10150 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10151 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10152 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10153 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10154 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10155 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10156 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10157 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10158 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10159 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10160 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10161 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10162 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10163 ) != 0
10164 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10165 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10166 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10167 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10168 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10169 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10170 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10171 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10172 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10173 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10174 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10175 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10176 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10177 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10178 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10179 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10180 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10181 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10182 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10183 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10184 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10185 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10186 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10187 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10188 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10189 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10190 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10191 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10192 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10193 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10194 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10195 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10196 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10197 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10198 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10199 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10200 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10201 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10202 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10203 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10204 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10205 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10206 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10207 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10208 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10209 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10210 ) != 0
10211 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10212 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10213 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10214 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10215 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10216 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10217 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10218 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10219 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10220 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10221 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10222 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10223 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10224 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10225 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10226 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10227 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10228 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10229 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10230 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10231 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10232 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10233 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10234 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10235 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10236 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10237 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10238 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10239 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10240 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10241 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10242 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10243 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10244 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10245 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10246 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10247 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10248 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10249 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10250 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10251 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10252 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10253 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10254 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10255 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10256 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10257 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10258 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10259 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10260 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10261 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10262 ) != 0;
10263}
10264
10265
10266/**
10267 * Runs the guest code using VT-x.
10268 *
10269 * @returns Strict VBox status code (i.e. informational status codes too).
10270 * @param pVCpu The cross context virtual CPU structure.
10271 */
10272VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
10273{
10274 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10275 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10276 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10277 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
10278
10279 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10280
10281 VBOXSTRICTRC rcStrict;
10282 if ( !pVCpu->hm.s.fUseDebugLoop
10283 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10284 && !DBGFIsStepping(pVCpu)
10285 && !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
10286 rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu);
10287 else
10288 rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu);
10289
10290 if (rcStrict == VERR_EM_INTERPRETER)
10291 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10292 else if (rcStrict == VINF_EM_RESET)
10293 rcStrict = VINF_EM_TRIPLE_FAULT;
10294
10295 int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
10296 if (RT_FAILURE(rc2))
10297 {
10298 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10299 rcStrict = rc2;
10300 }
10301 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10302 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10303 return rcStrict;
10304}
10305
10306
10307#ifndef HMVMX_USE_FUNCTION_TABLE
10308DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10309{
10310#ifdef DEBUG_ramshankar
10311#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10312 do { \
10313 if (a_fSave != 0) \
10314 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10315 VBOXSTRICTRC rcStrict = a_CallExpr; \
10316 if (a_fSave != 0) \
10317 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10318 return rcStrict; \
10319 } while (0)
10320#else
10321# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10322#endif
10323 switch (rcReason)
10324 {
10325 case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
10326 case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
10327 case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
10328 case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
10329 case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
10330 case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
10331 case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
10332 case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
10333 case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
10334 case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
10335 case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
10336 case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
10337 case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
10338 case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
10339 case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
10340 case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
10341 case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
10342 case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
10343 case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
10344 case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
10345 case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
10346 case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
10347 case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
10348 case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pVmxTransient));
10349 case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
10350 case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
10351 case VMX_EXIT_GDTR_IDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10352 case VMX_EXIT_LDTR_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10353 case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
10354 case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
10355 case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
10356 case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
10357 case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
10358 case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
10359
10360 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
10361 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
10362 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pVmxTransient);
10363 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pVmxTransient);
10364 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pVmxTransient);
10365 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pVmxTransient);
10366 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pVmxTransient);
10367 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
10368 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pVmxTransient);
10369
10370 case VMX_EXIT_VMCLEAR:
10371 case VMX_EXIT_VMLAUNCH:
10372 case VMX_EXIT_VMPTRLD:
10373 case VMX_EXIT_VMPTRST:
10374 case VMX_EXIT_VMREAD:
10375 case VMX_EXIT_VMRESUME:
10376 case VMX_EXIT_VMWRITE:
10377 case VMX_EXIT_VMXOFF:
10378 case VMX_EXIT_VMXON:
10379 case VMX_EXIT_INVEPT:
10380 case VMX_EXIT_INVVPID:
10381 case VMX_EXIT_VMFUNC:
10382 case VMX_EXIT_XSAVES:
10383 case VMX_EXIT_XRSTORS:
10384 return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
10385
10386 case VMX_EXIT_ENCLS:
10387 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10388 case VMX_EXIT_PML_FULL:
10389 default:
10390 return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
10391 }
10392#undef VMEXIT_CALL_RET
10393}
10394#endif /* !HMVMX_USE_FUNCTION_TABLE */
10395
10396
10397#ifdef VBOX_STRICT
10398/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10399# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10400 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10401
10402# define HMVMX_ASSERT_PREEMPT_CPUID() \
10403 do { \
10404 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10405 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10406 } while (0)
10407
10408# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10409 do { \
10410 AssertPtr((a_pVCpu)); \
10411 AssertPtr((a_pVmxTransient)); \
10412 Assert((a_pVmxTransient)->fVMEntryFailed == false); \
10413 Assert(ASMIntAreEnabled()); \
10414 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10415 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10416 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
10417 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10418 if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
10419 HMVMX_ASSERT_PREEMPT_CPUID(); \
10420 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10421 } while (0)
10422
10423# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10424 do { \
10425 Log4Func(("\n")); \
10426 } while (0)
10427#else
10428# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10429 do { \
10430 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10431 NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
10432 } while (0)
10433# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
10434#endif
10435
10436
10437/**
10438 * Advances the guest RIP by the specified number of bytes.
10439 *
10440 * @param pVCpu The cross context virtual CPU structure.
10441 * @param cbInstr Number of bytes to advance the RIP by.
10442 *
10443 * @remarks No-long-jump zone!!!
10444 */
10445DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
10446{
10447 /* Advance the RIP. */
10448 pVCpu->cpum.GstCtx.rip += cbInstr;
10449 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10450
10451 /* Update interrupt inhibition. */
10452 if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10453 && pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
10454 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10455}
10456
10457
10458/**
10459 * Advances the guest RIP after reading it from the VMCS.
10460 *
10461 * @returns VBox status code, no informational status codes.
10462 * @param pVCpu The cross context virtual CPU structure.
10463 * @param pVmxTransient Pointer to the VMX transient structure.
10464 *
10465 * @remarks No-long-jump zone!!!
10466 */
10467static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10468{
10469 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10470 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS);
10471 AssertRCReturn(rc, rc);
10472
10473 hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
10474 return VINF_SUCCESS;
10475}
10476
10477
10478/**
10479 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10480 * and update error record fields accordingly.
10481 *
10482 * @return VMX_IGS_* return codes.
10483 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10484 * wrong with the guest state.
10485 *
10486 * @param pVCpu The cross context virtual CPU structure.
10487 *
10488 * @remarks This function assumes our cache of the VMCS controls
10489 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10490 */
10491static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu)
10492{
10493#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10494#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10495 uError = (err); \
10496 break; \
10497 } else do { } while (0)
10498
10499 int rc;
10500 PVM pVM = pVCpu->CTX_SUFF(pVM);
10501 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10502 uint32_t uError = VMX_IGS_ERROR;
10503 uint32_t u32Val;
10504 bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10505
10506 do
10507 {
10508 /*
10509 * CR0.
10510 */
10511 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10512 uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10513 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10514 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10515 if (fUnrestrictedGuest)
10516 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
10517
10518 uint32_t u32GuestCr0;
10519 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10520 AssertRCBreak(rc);
10521 HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10522 HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10523 if ( !fUnrestrictedGuest
10524 && (u32GuestCr0 & X86_CR0_PG)
10525 && !(u32GuestCr0 & X86_CR0_PE))
10526 {
10527 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10528 }
10529
10530 /*
10531 * CR4.
10532 */
10533 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10534 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10535
10536 uint32_t u32GuestCr4;
10537 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10538 AssertRCBreak(rc);
10539 HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10540 HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10541
10542 /*
10543 * IA32_DEBUGCTL MSR.
10544 */
10545 uint64_t u64Val;
10546 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10547 AssertRCBreak(rc);
10548 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10549 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10550 {
10551 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10552 }
10553 uint64_t u64DebugCtlMsr = u64Val;
10554
10555#ifdef VBOX_STRICT
10556 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10557 AssertRCBreak(rc);
10558 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10559#endif
10560 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
10561
10562 /*
10563 * RIP and RFLAGS.
10564 */
10565 uint32_t u32Eflags;
10566#if HC_ARCH_BITS == 64
10567 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10568 AssertRCBreak(rc);
10569 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10570 if ( !fLongModeGuest
10571 || !pCtx->cs.Attr.n.u1Long)
10572 {
10573 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10574 }
10575 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10576 * must be identical if the "IA-32e mode guest" VM-entry
10577 * control is 1 and CS.L is 1. No check applies if the
10578 * CPU supports 64 linear-address bits. */
10579
10580 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10581 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10582 AssertRCBreak(rc);
10583 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10584 VMX_IGS_RFLAGS_RESERVED);
10585 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10586 u32Eflags = u64Val;
10587#else
10588 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10589 AssertRCBreak(rc);
10590 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10591 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10592#endif
10593
10594 if ( fLongModeGuest
10595 || ( fUnrestrictedGuest
10596 && !(u32GuestCr0 & X86_CR0_PE)))
10597 {
10598 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10599 }
10600
10601 uint32_t u32EntryInfo;
10602 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10603 AssertRCBreak(rc);
10604 if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
10605 && VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
10606 {
10607 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10608 }
10609
10610 /*
10611 * 64-bit checks.
10612 */
10613#if HC_ARCH_BITS == 64
10614 if (fLongModeGuest)
10615 {
10616 HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10617 HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10618 }
10619
10620 if ( !fLongModeGuest
10621 && (u32GuestCr4 & X86_CR4_PCIDE))
10622 {
10623 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10624 }
10625
10626 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10627 * 51:32 beyond the processor's physical-address width are 0. */
10628
10629 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10630 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10631 {
10632 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10633 }
10634
10635 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10636 AssertRCBreak(rc);
10637 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10638
10639 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10640 AssertRCBreak(rc);
10641 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10642#endif
10643
10644 /*
10645 * PERF_GLOBAL MSR.
10646 */
10647 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
10648 {
10649 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10650 AssertRCBreak(rc);
10651 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10652 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10653 }
10654
10655 /*
10656 * PAT MSR.
10657 */
10658 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
10659 {
10660 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10661 AssertRCBreak(rc);
10662 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10663 for (unsigned i = 0; i < 8; i++)
10664 {
10665 uint8_t u8Val = (u64Val & 0xff);
10666 if ( u8Val != 0 /* UC */
10667 && u8Val != 1 /* WC */
10668 && u8Val != 4 /* WT */
10669 && u8Val != 5 /* WP */
10670 && u8Val != 6 /* WB */
10671 && u8Val != 7 /* UC- */)
10672 {
10673 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10674 }
10675 u64Val >>= 8;
10676 }
10677 }
10678
10679 /*
10680 * EFER MSR.
10681 */
10682 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
10683 {
10684 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10685 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10686 AssertRCBreak(rc);
10687 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10688 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10689 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10690 & VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
10691 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10692 /** @todo r=ramshankar: Unrestricted check here is probably wrong, see
10693 * iemVmxVmentryCheckGuestState(). */
10694 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10695 || !(u32GuestCr0 & X86_CR0_PG)
10696 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10697 VMX_IGS_EFER_LMA_LME_MISMATCH);
10698 }
10699
10700 /*
10701 * Segment registers.
10702 */
10703 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10704 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10705 if (!(u32Eflags & X86_EFL_VM))
10706 {
10707 /* CS */
10708 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10709 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10710 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10711 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10712 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10713 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10714 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10715 /* CS cannot be loaded with NULL in protected mode. */
10716 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10717 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10718 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10719 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10720 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10721 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10722 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10723 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10724 else
10725 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10726
10727 /* SS */
10728 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10729 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10730 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10731 if ( !(pCtx->cr0 & X86_CR0_PE)
10732 || pCtx->cs.Attr.n.u4Type == 3)
10733 {
10734 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10735 }
10736 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10737 {
10738 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10739 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10740 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10741 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10742 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10743 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10744 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10745 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10746 }
10747
10748 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmenReg(). */
10749 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10750 {
10751 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10752 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10753 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10754 || pCtx->ds.Attr.n.u4Type > 11
10755 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10756 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10757 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10758 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10759 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10760 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10761 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10762 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10763 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10764 }
10765 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10766 {
10767 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10768 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10769 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10770 || pCtx->es.Attr.n.u4Type > 11
10771 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10772 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10773 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10774 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10775 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10776 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10777 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10778 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10779 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10780 }
10781 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10782 {
10783 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10784 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10785 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10786 || pCtx->fs.Attr.n.u4Type > 11
10787 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10788 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10789 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10790 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10791 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10792 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10793 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10794 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10795 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10796 }
10797 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10798 {
10799 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10800 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10801 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10802 || pCtx->gs.Attr.n.u4Type > 11
10803 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10804 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10805 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10806 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10807 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10808 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10809 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10810 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10811 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10812 }
10813 /* 64-bit capable CPUs. */
10814#if HC_ARCH_BITS == 64
10815 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10816 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10817 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10818 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10819 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10820 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10821 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10822 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10823 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10824 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10825 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10826#endif
10827 }
10828 else
10829 {
10830 /* V86 mode checks. */
10831 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10832 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10833 {
10834 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10835 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10836 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10837 }
10838 else
10839 {
10840 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10841 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10842 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10843 }
10844
10845 /* CS */
10846 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10847 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10848 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10849 /* SS */
10850 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10851 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10852 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10853 /* DS */
10854 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10855 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10856 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10857 /* ES */
10858 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10859 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10860 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10861 /* FS */
10862 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10863 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10864 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10865 /* GS */
10866 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10867 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10868 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10869 /* 64-bit capable CPUs. */
10870#if HC_ARCH_BITS == 64
10871 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10872 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10873 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10874 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10875 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10876 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10877 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10878 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10879 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10880 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10881 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10882#endif
10883 }
10884
10885 /*
10886 * TR.
10887 */
10888 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10889 /* 64-bit capable CPUs. */
10890#if HC_ARCH_BITS == 64
10891 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10892#endif
10893 if (fLongModeGuest)
10894 {
10895 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10896 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10897 }
10898 else
10899 {
10900 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10901 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10902 VMX_IGS_TR_ATTR_TYPE_INVALID);
10903 }
10904 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10905 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10906 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10907 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10908 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10909 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10910 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10911 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10912
10913 /*
10914 * GDTR and IDTR.
10915 */
10916#if HC_ARCH_BITS == 64
10917 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10918 AssertRCBreak(rc);
10919 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10920
10921 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10922 AssertRCBreak(rc);
10923 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10924#endif
10925
10926 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10927 AssertRCBreak(rc);
10928 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10929
10930 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10931 AssertRCBreak(rc);
10932 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10933
10934 /*
10935 * Guest Non-Register State.
10936 */
10937 /* Activity State. */
10938 uint32_t u32ActivityState;
10939 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10940 AssertRCBreak(rc);
10941 HMVMX_CHECK_BREAK( !u32ActivityState
10942 || (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
10943 VMX_IGS_ACTIVITY_STATE_INVALID);
10944 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10945 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10946 uint32_t u32IntrState;
10947 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
10948 AssertRCBreak(rc);
10949 if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
10950 || u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10951 {
10952 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10953 }
10954
10955 /** @todo Activity state and injecting interrupts. Left as a todo since we
10956 * currently don't use activity states but ACTIVE. */
10957
10958 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10959 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10960
10961 /* Guest interruptibility-state. */
10962 HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10963 HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
10964 != (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10965 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10966 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10967 || !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10968 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10969 if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
10970 {
10971 if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
10972 {
10973 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10974 && !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10975 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10976 }
10977 else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
10978 {
10979 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10980 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10981 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10982 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10983 }
10984 }
10985 /** @todo Assumes the processor is not in SMM. */
10986 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10987 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10988 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10989 || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10990 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10991 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
10992 && VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
10993 && VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
10994 {
10995 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
10996 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10997 }
10998
10999 /* Pending debug exceptions. */
11000#if HC_ARCH_BITS == 64
11001 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
11002 AssertRCBreak(rc);
11003 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11004 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11005 u32Val = u64Val; /* For pending debug exceptions checks below. */
11006#else
11007 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
11008 AssertRCBreak(rc);
11009 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11010 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11011#endif
11012
11013 if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
11014 || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
11015 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11016 {
11017 if ( (u32Eflags & X86_EFL_TF)
11018 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11019 {
11020 /* Bit 14 is PendingDebug.BS. */
11021 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11022 }
11023 if ( !(u32Eflags & X86_EFL_TF)
11024 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11025 {
11026 /* Bit 14 is PendingDebug.BS. */
11027 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11028 }
11029 }
11030
11031 /* VMCS link pointer. */
11032 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11033 AssertRCBreak(rc);
11034 if (u64Val != UINT64_C(0xffffffffffffffff))
11035 {
11036 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11037 /** @todo Bits beyond the processor's physical-address width MBZ. */
11038 /** @todo 32-bit located in memory referenced by value of this field (as a
11039 * physical address) must contain the processor's VMCS revision ID. */
11040 /** @todo SMM checks. */
11041 }
11042
11043 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11044 * not using Nested Paging? */
11045 if ( pVM->hm.s.fNestedPaging
11046 && !fLongModeGuest
11047 && CPUMIsGuestInPAEModeEx(pCtx))
11048 {
11049 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11050 AssertRCBreak(rc);
11051 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11052
11053 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11054 AssertRCBreak(rc);
11055 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11056
11057 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11058 AssertRCBreak(rc);
11059 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11060
11061 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11062 AssertRCBreak(rc);
11063 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11064 }
11065
11066 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11067 if (uError == VMX_IGS_ERROR)
11068 uError = VMX_IGS_REASON_NOT_FOUND;
11069 } while (0);
11070
11071 pVCpu->hm.s.u32HMError = uError;
11072 return uError;
11073
11074#undef HMVMX_ERROR_BREAK
11075#undef HMVMX_CHECK_BREAK
11076}
11077
11078
11079/** @name VM-exit handlers.
11080 * @{
11081 */
11082/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11083/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11084/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11085
11086/**
11087 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11088 */
11089HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11090{
11091 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11092 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11093 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11094 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11095 return VINF_SUCCESS;
11096 return VINF_EM_RAW_INTERRUPT;
11097}
11098
11099
11100/**
11101 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11102 */
11103HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11104{
11105 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11106 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11107
11108 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11109 AssertRCReturn(rc, rc);
11110
11111 uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
11112 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
11113 && uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
11114 Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11115
11116 if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
11117 {
11118 /*
11119 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
11120 * injected it ourselves and anything we inject is not going to cause a VM-exit directly
11121 * for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
11122 *
11123 * [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11124 * [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
11125 */
11126 VMXDispatchHostNmi();
11127 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11128 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11129 return VINF_SUCCESS;
11130 }
11131
11132 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11133 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
11134 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11135 { /* likely */ }
11136 else
11137 {
11138 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11139 rcStrictRc1 = VINF_SUCCESS;
11140 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11141 return rcStrictRc1;
11142 }
11143
11144 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11145 uint32_t uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
11146 switch (uIntType)
11147 {
11148 case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11149 Assert(uVector == X86_XCPT_DB);
11150 RT_FALL_THRU();
11151 case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11152 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
11153 RT_FALL_THRU();
11154 case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11155 {
11156 /*
11157 * If there's any exception caused as a result of event injection, the resulting
11158 * secondary/final execption will be pending, we shall continue guest execution
11159 * after injecting the event. The page-fault case is complicated and we manually
11160 * handle any currently pending event in hmR0VmxExitXcptPF.
11161 */
11162 if (!pVCpu->hm.s.Event.fPending)
11163 { /* likely */ }
11164 else if (uVector != X86_XCPT_PF)
11165 {
11166 rc = VINF_SUCCESS;
11167 break;
11168 }
11169
11170 switch (uVector)
11171 {
11172 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
11173 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
11174 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
11175 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
11176 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
11177 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
11178
11179 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11180 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11181 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11182 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11183 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11184 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11185 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11186 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11187 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11188 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11189 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11190 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11191 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11192 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11193 default:
11194 {
11195 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11196 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11197 {
11198 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11199 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11200 Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
11201
11202 rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0);
11203 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11204 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11205 AssertRCReturn(rc, rc);
11206 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11207 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11208 0 /* GCPtrFaultAddress */);
11209 }
11210 else
11211 {
11212 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11213 pVCpu->hm.s.u32HMError = uVector;
11214 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11215 }
11216 break;
11217 }
11218 }
11219 break;
11220 }
11221
11222 default:
11223 {
11224 pVCpu->hm.s.u32HMError = uExitIntInfo;
11225 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11226 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
11227 break;
11228 }
11229 }
11230 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11231 return rc;
11232}
11233
11234
11235/**
11236 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11237 */
11238HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11239{
11240 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11241
11242 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11243 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11244
11245 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11246 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11247 return VINF_SUCCESS;
11248}
11249
11250
11251/**
11252 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11253 */
11254HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11255{
11256 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11257 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)))
11258 {
11259 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11260 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11261 }
11262
11263 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS));
11264
11265 /*
11266 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11267 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11268 */
11269 uint32_t fIntrState = 0;
11270 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
11271 AssertRCReturn(rc, rc);
11272 Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
11273 if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
11274 {
11275 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11276 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11277
11278 fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
11279 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
11280 AssertRCReturn(rc, rc);
11281 }
11282
11283 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11284 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11285
11286 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11287 return VINF_SUCCESS;
11288}
11289
11290
11291/**
11292 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11293 */
11294HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11295{
11296 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11297 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11298}
11299
11300
11301/**
11302 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11303 */
11304HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11305{
11306 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11307 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11308}
11309
11310
11311/**
11312 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11313 */
11314HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11315{
11316 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11317
11318 /*
11319 * Get the state we need and update the exit history entry.
11320 */
11321 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11322 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
11323 AssertRCReturn(rc, rc);
11324
11325 VBOXSTRICTRC rcStrict;
11326 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11327 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
11328 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11329 if (!pExitRec)
11330 {
11331 /*
11332 * Regular CPUID instruction execution.
11333 */
11334 rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
11335 if (rcStrict == VINF_SUCCESS)
11336 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11337 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11338 {
11339 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11340 rcStrict = VINF_SUCCESS;
11341 }
11342 }
11343 else
11344 {
11345 /*
11346 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
11347 */
11348 int rc2 = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11349 AssertRCReturn(rc2, rc2);
11350
11351 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11352 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11353
11354 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11355 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11356
11357 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11358 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11359 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11360 }
11361 return rcStrict;
11362}
11363
11364
11365/**
11366 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11367 */
11368HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11369{
11370 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11371 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR4);
11372 AssertRCReturn(rc, rc);
11373
11374 if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
11375 return VINF_EM_RAW_EMULATE_INSTR;
11376
11377 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11378 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11379}
11380
11381
11382/**
11383 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11384 */
11385HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11386{
11387 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11388 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11389 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11390 AssertRCReturn(rc, rc);
11391
11392 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11393 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11394 {
11395 /* If we get a spurious VM-exit when offsetting is enabled,
11396 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11397 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
11398 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11399 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11400 }
11401 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11402 {
11403 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11404 rcStrict = VINF_SUCCESS;
11405 }
11406 return rcStrict;
11407}
11408
11409
11410/**
11411 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11412 */
11413HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11414{
11415 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11416 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_TSC_AUX);
11417 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11418 AssertRCReturn(rc, rc);
11419
11420 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11421 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11422 {
11423 /* If we get a spurious VM-exit when offsetting is enabled,
11424 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11425 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
11426 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11427 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11428 }
11429 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11430 {
11431 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11432 rcStrict = VINF_SUCCESS;
11433 }
11434 return rcStrict;
11435}
11436
11437
11438/**
11439 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11440 */
11441HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11442{
11443 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11444 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11445 AssertRCReturn(rc, rc);
11446
11447 PVM pVM = pVCpu->CTX_SUFF(pVM);
11448 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11449 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11450 if (RT_LIKELY(rc == VINF_SUCCESS))
11451 {
11452 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11453 Assert(pVmxTransient->cbInstr == 2);
11454 }
11455 else
11456 {
11457 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11458 rc = VERR_EM_INTERPRETER;
11459 }
11460 return rc;
11461}
11462
11463
11464/**
11465 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11466 */
11467HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11468{
11469 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11470
11471 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11472 if (EMAreHypercallInstructionsEnabled(pVCpu))
11473 {
11474 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CR0
11475 | CPUMCTX_EXTRN_SS | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_EFER);
11476 AssertRCReturn(rc, rc);
11477
11478 /* Perform the hypercall. */
11479 rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
11480 if (rcStrict == VINF_SUCCESS)
11481 {
11482 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11483 AssertRCReturn(rc, rc);
11484 }
11485 else
11486 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11487 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11488 || RT_FAILURE(rcStrict));
11489
11490 /* If the hypercall changes anything other than guest's general-purpose registers,
11491 we would need to reload the guest changed bits here before VM-entry. */
11492 }
11493 else
11494 Log4Func(("Hypercalls not enabled\n"));
11495
11496 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11497 if (RT_FAILURE(rcStrict))
11498 {
11499 hmR0VmxSetPendingXcptUD(pVCpu);
11500 rcStrict = VINF_SUCCESS;
11501 }
11502
11503 return rcStrict;
11504}
11505
11506
11507/**
11508 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11509 */
11510HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11511{
11512 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11513 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11514
11515 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11516 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11517 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
11518 AssertRCReturn(rc, rc);
11519
11520 VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
11521
11522 if (rcStrict == VINF_SUCCESS || rcStrict == VINF_PGM_SYNC_CR3)
11523 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11524 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11525 {
11526 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11527 rcStrict = VINF_SUCCESS;
11528 }
11529 else
11530 AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) sttus: %Rrc\n", pVmxTransient->uExitQual,
11531 VBOXSTRICTRC_VAL(rcStrict)));
11532 return rcStrict;
11533}
11534
11535
11536/**
11537 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11538 */
11539HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11540{
11541 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11542 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11543 AssertRCReturn(rc, rc);
11544
11545 PVM pVM = pVCpu->CTX_SUFF(pVM);
11546 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11547 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11548 if (RT_LIKELY(rc == VINF_SUCCESS))
11549 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11550 else
11551 {
11552 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11553 rc = VERR_EM_INTERPRETER;
11554 }
11555 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11556 return rc;
11557}
11558
11559
11560/**
11561 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11562 */
11563HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11564{
11565 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11566 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11567 AssertRCReturn(rc, rc);
11568
11569 PVM pVM = pVCpu->CTX_SUFF(pVM);
11570 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11571 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11572 rc = VBOXSTRICTRC_VAL(rc2);
11573 if (RT_LIKELY( rc == VINF_SUCCESS
11574 || rc == VINF_EM_HALT))
11575 {
11576 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11577 AssertRCReturn(rc3, rc3);
11578
11579 if ( rc == VINF_EM_HALT
11580 && EMMonitorWaitShouldContinue(pVCpu, pCtx))
11581 rc = VINF_SUCCESS;
11582 }
11583 else
11584 {
11585 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11586 rc = VERR_EM_INTERPRETER;
11587 }
11588 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11589 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11590 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11591 return rc;
11592}
11593
11594
11595/**
11596 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11597 */
11598HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11599{
11600 /*
11601 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11602 * mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11603 * treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11604 * VMX root operation. If we get here, something funny is going on.
11605 *
11606 * See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11607 */
11608 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11609 AssertMsgFailed(("Unexpected RSM VM-exit\n"));
11610 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11611}
11612
11613
11614/**
11615 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11616 */
11617HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11618{
11619 /*
11620 * This can only happen if we support dual-monitor treatment of SMI, which can be activated
11621 * by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11622 * this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11623 * an SMI. If we get here, something funny is going on.
11624 *
11625 * See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11626 * See Intel spec. 25.3 "Other Causes of VM-Exits"
11627 */
11628 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11629 AssertMsgFailed(("Unexpected SMI VM-exit\n"));
11630 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11631}
11632
11633
11634/**
11635 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11636 */
11637HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11638{
11639 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11640 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11641 AssertMsgFailed(("Unexpected IO SMI VM-exit\n"));
11642 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11643}
11644
11645
11646/**
11647 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11648 */
11649HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11650{
11651 /*
11652 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11653 * We don't make use of it as our guests don't have direct access to the host LAPIC.
11654 * See Intel spec. 25.3 "Other Causes of VM-exits".
11655 */
11656 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11657 AssertMsgFailed(("Unexpected SIPI VM-exit\n"));
11658 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11659}
11660
11661
11662/**
11663 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11664 * VM-exit.
11665 */
11666HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11667{
11668 /*
11669 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11670 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11671 *
11672 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11673 * See Intel spec. "23.8 Restrictions on VMX operation".
11674 */
11675 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11676 return VINF_SUCCESS;
11677}
11678
11679
11680/**
11681 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11682 * VM-exit.
11683 */
11684HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11685{
11686 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11687 return VINF_EM_RESET;
11688}
11689
11690
11691/**
11692 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11693 */
11694HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11695{
11696 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11697 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_HLT_EXIT);
11698
11699 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11700 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RFLAGS);
11701 AssertRCReturn(rc, rc);
11702
11703 if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
11704 rc = VINF_SUCCESS;
11705 else
11706 rc = VINF_EM_HALT;
11707
11708 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11709 if (rc != VINF_SUCCESS)
11710 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11711 return rc;
11712}
11713
11714
11715/**
11716 * VM-exit handler for instructions that result in a \#UD exception delivered to
11717 * the guest.
11718 */
11719HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11720{
11721 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11722 hmR0VmxSetPendingXcptUD(pVCpu);
11723 return VINF_SUCCESS;
11724}
11725
11726
11727/**
11728 * VM-exit handler for expiry of the VMX preemption timer.
11729 */
11730HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11731{
11732 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11733
11734 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11735 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11736
11737 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11738 PVM pVM = pVCpu->CTX_SUFF(pVM);
11739 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11740 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11741 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11742}
11743
11744
11745/**
11746 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11747 */
11748HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11749{
11750 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11751
11752 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11753 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_CR4);
11754 AssertRCReturn(rc, rc);
11755
11756 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11757 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11758 : HM_CHANGED_RAISED_XCPT_MASK);
11759
11760 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11761 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
11762
11763 return rcStrict;
11764}
11765
11766
11767/**
11768 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11769 */
11770HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11771{
11772 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11773 /** @todo Use VM-exit instruction information. */
11774 return VERR_EM_INTERPRETER;
11775}
11776
11777
11778/**
11779 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11780 * Error VM-exit.
11781 */
11782HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11783{
11784 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11785 AssertRCReturn(rc, rc);
11786 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11787 if (RT_FAILURE(rc))
11788 return rc;
11789
11790 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
11791 NOREF(uInvalidReason);
11792
11793#ifdef VBOX_STRICT
11794 uint32_t fIntrState;
11795 RTHCUINTREG uHCReg;
11796 uint64_t u64Val;
11797 uint32_t u32Val;
11798
11799 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11800 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11801 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11802 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
11803 AssertRCReturn(rc, rc);
11804
11805 Log4(("uInvalidReason %u\n", uInvalidReason));
11806 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11807 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11808 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11809 Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
11810
11811 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11812 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11813 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11814 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11815 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11816 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11817 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11818 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11819 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11820 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11821 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11822 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11823
11824 hmR0DumpRegs(pVCpu);
11825#else
11826 NOREF(pVmxTransient);
11827#endif
11828
11829 return VERR_VMX_INVALID_GUEST_STATE;
11830}
11831
11832
11833/**
11834 * VM-exit handler for VM-entry failure due to an MSR-load
11835 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11836 */
11837HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11838{
11839 AssertMsgFailed(("Unexpected MSR-load exit\n"));
11840 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11841}
11842
11843
11844/**
11845 * VM-exit handler for VM-entry failure due to a machine-check event
11846 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11847 */
11848HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11849{
11850 AssertMsgFailed(("Unexpected machine-check event exit\n"));
11851 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11852}
11853
11854
11855/**
11856 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11857 * theory.
11858 */
11859HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11860{
11861 RT_NOREF2(pVCpu, pVmxTransient);
11862 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d\n", pVmxTransient->uExitReason));
11863 return VERR_VMX_UNDEFINED_EXIT_CODE;
11864}
11865
11866
11867/**
11868 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11869 * (VMX_EXIT_GDTR_IDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11870 * Conditional VM-exit.
11871 */
11872HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11873{
11874 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11875
11876 /* By default, we don't enable VMX_PROC_CTLS2_DESCRIPTOR_TABLE_EXIT. */
11877 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11878 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_DESC_TABLE_EXIT)
11879 return VERR_EM_INTERPRETER;
11880 AssertMsgFailed(("Unexpected XDTR access\n"));
11881 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11882}
11883
11884
11885/**
11886 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11887 */
11888HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11889{
11890 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11891
11892 /* By default, we don't enable VMX_PROC_CTLS2_RDRAND_EXIT. */
11893 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_RDRAND_EXIT)
11894 return VERR_EM_INTERPRETER;
11895 AssertMsgFailed(("Unexpected RDRAND exit\n"));
11896 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11897}
11898
11899
11900/**
11901 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11902 */
11903HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11904{
11905 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11906
11907 /** @todo Optimize this: We currently drag in in the whole MSR state
11908 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11909 * MSRs required. That would require changes to IEM and possibly CPUM too.
11910 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11911 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
11912 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11913 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
11914 switch (idMsr)
11915 {
11916 /* The FS and GS base MSRs are not part of the above all-MSRs mask. */
11917 case MSR_K8_FS_BASE: rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_FS); break;
11918 case MSR_K8_GS_BASE: rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_GS); break;
11919 }
11920 AssertRCReturn(rc, rc);
11921
11922 Log4Func(("ecx=%#RX32\n", idMsr));
11923
11924#ifdef VBOX_STRICT
11925 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
11926 {
11927 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr)
11928 && idMsr != MSR_K6_EFER)
11929 {
11930 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
11931 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11932 }
11933 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11934 {
11935 VMXMSREXITREAD enmRead;
11936 VMXMSREXITWRITE enmWrite;
11937 int rc2 = HMVmxGetMsrPermission(pVCpu->hm.s.vmx.pvMsrBitmap, idMsr, &enmRead, &enmWrite);
11938 AssertRCReturn(rc2, rc2);
11939 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11940 {
11941 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
11942 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11943 }
11944 }
11945 }
11946#endif
11947
11948 VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
11949 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11950 if (rcStrict == VINF_SUCCESS)
11951 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11952 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX);
11953 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11954 {
11955 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11956 rcStrict = VINF_SUCCESS;
11957 }
11958 else
11959 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11960
11961 return rcStrict;
11962}
11963
11964
11965/**
11966 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11967 */
11968HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11969{
11970 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11971
11972 /** @todo Optimize this: We currently drag in in the whole MSR state
11973 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11974 * MSRs required. That would require changes to IEM and possibly CPUM too.
11975 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11976 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
11977 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11978 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK
11979 | CPUMCTX_EXTRN_ALL_MSRS);
11980 switch (idMsr)
11981 {
11982 /*
11983 * The FS and GS base MSRs are not part of the above all-MSRs mask.
11984 *
11985 * Although we don't need to fetch the base as it will be overwritten shortly, while
11986 * loading guest-state we would also load the entire segment register including limit
11987 * and attributes and thus we need to load them here.
11988 */
11989 case MSR_K8_FS_BASE: rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_FS); break;
11990 case MSR_K8_GS_BASE: rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_GS); break;
11991 }
11992 AssertRCReturn(rc, rc);
11993
11994 Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
11995
11996 VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
11997 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
11998
11999 if (rcStrict == VINF_SUCCESS)
12000 {
12001 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12002
12003 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
12004 if ( idMsr == MSR_IA32_APICBASE
12005 || ( idMsr >= MSR_IA32_X2APIC_START
12006 && idMsr <= MSR_IA32_X2APIC_END))
12007 {
12008 /*
12009 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12010 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before IEM changes it.
12011 */
12012 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
12013 }
12014 else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12015 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12016 else if (idMsr == MSR_K6_EFER)
12017 {
12018 /*
12019 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12020 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12021 * the other bits as well, SCE and NXE. See @bugref{7368}.
12022 */
12023 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS
12024 | HM_CHANGED_VMX_EXIT_CTLS);
12025 }
12026
12027 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12028 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
12029 {
12030 switch (idMsr)
12031 {
12032 case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
12033 case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
12034 case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
12035 case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
12036 case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
12037 case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
12038 default:
12039 {
12040 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
12041 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12042 else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
12043 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
12044 break;
12045 }
12046 }
12047 }
12048#ifdef VBOX_STRICT
12049 else
12050 {
12051 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12052 switch (idMsr)
12053 {
12054 case MSR_IA32_SYSENTER_CS:
12055 case MSR_IA32_SYSENTER_EIP:
12056 case MSR_IA32_SYSENTER_ESP:
12057 case MSR_K8_FS_BASE:
12058 case MSR_K8_GS_BASE:
12059 {
12060 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
12061 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12062 }
12063
12064 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12065 default:
12066 {
12067 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
12068 {
12069 /* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
12070 if (idMsr != MSR_K6_EFER)
12071 {
12072 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12073 idMsr));
12074 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12075 }
12076 }
12077
12078 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
12079 {
12080 VMXMSREXITREAD enmRead;
12081 VMXMSREXITWRITE enmWrite;
12082 int rc2 = HMVmxGetMsrPermission(pVCpu->hm.s.vmx.pvMsrBitmap, idMsr, &enmRead, &enmWrite);
12083 AssertRCReturn(rc2, rc2);
12084 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12085 {
12086 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
12087 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12088 }
12089 }
12090 break;
12091 }
12092 }
12093 }
12094#endif /* VBOX_STRICT */
12095 }
12096 else if (rcStrict == VINF_IEM_RAISED_XCPT)
12097 {
12098 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
12099 rcStrict = VINF_SUCCESS;
12100 }
12101 else
12102 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12103
12104 return rcStrict;
12105}
12106
12107
12108/**
12109 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12110 */
12111HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12112{
12113 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12114 /** @todo The guest has likely hit a contended spinlock. We might want to
12115 * poke a schedule different guest VCPU. */
12116 return VINF_EM_RAW_INTERRUPT;
12117}
12118
12119
12120/**
12121 * VM-exit handler for when the TPR value is lowered below the specified
12122 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12123 */
12124HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12125{
12126 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12127 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
12128
12129 /*
12130 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12131 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12132 */
12133 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12134 return VINF_SUCCESS;
12135}
12136
12137
12138/**
12139 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12140 * VM-exit.
12141 *
12142 * @retval VINF_SUCCESS when guest execution can continue.
12143 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12144 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12145 * interpreter.
12146 */
12147HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12148{
12149 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12150 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12151
12152 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12153 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12154 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12155 AssertRCReturn(rc, rc);
12156
12157 VBOXSTRICTRC rcStrict;
12158 PVM pVM = pVCpu->CTX_SUFF(pVM);
12159 RTGCUINTPTR const uExitQual = pVmxTransient->uExitQual;
12160 uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
12161 switch (uAccessType)
12162 {
12163 case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
12164 {
12165 uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
12166 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
12167 VMX_EXIT_QUAL_CRX_GENREG(uExitQual));
12168 AssertMsg( rcStrict == VINF_SUCCESS
12169 || rcStrict == VINF_IEM_RAISED_XCPT
12170 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12171
12172 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
12173 {
12174 case 0:
12175 {
12176 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12177 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12178 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
12179 Log4Func(("CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
12180
12181 /*
12182 * This is a kludge for handling switches back to real mode when we try to use
12183 * V86 mode to run real mode code directly. Problem is that V86 mode cannot
12184 * deal with special selector values, so we have to return to ring-3 and run
12185 * there till the selector values are V86 mode compatible.
12186 *
12187 * Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
12188 * latter is an alias for VINF_IEM_RAISED_XCPT which is converted to VINF_SUCCESs
12189 * at the end of this function.
12190 */
12191 if ( rc == VINF_SUCCESS
12192 && !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
12193 && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
12194 && (uOldCr0 & X86_CR0_PE)
12195 && !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
12196 {
12197 /** @todo check selectors rather than returning all the time. */
12198 Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
12199 rcStrict = VINF_EM_RESCHEDULE_REM;
12200 }
12201 break;
12202 }
12203
12204 case 2:
12205 {
12206 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
12207 /* Nothing to do here, CR2 it's not part of the VMCS. */
12208 break;
12209 }
12210
12211 case 3:
12212 {
12213 Assert( !pVM->hm.s.fNestedPaging
12214 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
12215 || pVCpu->hm.s.fUsingDebugLoop);
12216 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
12217 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12218 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR3);
12219 Log4Func(("CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
12220 break;
12221 }
12222
12223 case 4:
12224 {
12225 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
12226 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12227 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR4);
12228 Log4Func(("CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
12229 pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12230 break;
12231 }
12232
12233 case 8:
12234 {
12235 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
12236 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
12237 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12238 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_APIC_TPR);
12239 break;
12240 }
12241 default:
12242 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual)));
12243 break;
12244 }
12245 break;
12246 }
12247
12248 case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
12249 {
12250 Assert( !pVM->hm.s.fNestedPaging
12251 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
12252 || pVCpu->hm.s.fUsingDebugLoop
12253 || VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 3);
12254 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12255 Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 8
12256 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
12257
12258 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_GENREG(uExitQual),
12259 VMX_EXIT_QUAL_CRX_REGISTER(uExitQual));
12260 AssertMsg( rcStrict == VINF_SUCCESS
12261 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12262#ifdef VBOX_WITH_STATISTICS
12263 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
12264 {
12265 case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
12266 case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
12267 case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
12268 case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
12269 case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
12270 }
12271#endif
12272 Log4Func(("CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
12273 VBOXSTRICTRC_VAL(rcStrict)));
12274 if (VMX_EXIT_QUAL_CRX_GENREG(uExitQual) == X86_GREG_xSP)
12275 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RSP);
12276 else
12277 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12278 break;
12279 }
12280
12281 case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12282 {
12283 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12284 AssertMsg( rcStrict == VINF_SUCCESS
12285 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12286
12287 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12288 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12289 Log4Func(("CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12290 break;
12291 }
12292
12293 case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12294 {
12295 /* Note! LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here. */
12296 rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
12297 AssertRCReturn(rc, rc);
12298 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual),
12299 pVmxTransient->uGuestLinearAddr);
12300 AssertMsg( rcStrict == VINF_SUCCESS
12301 || rcStrict == VINF_IEM_RAISED_XCPT
12302 , ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12303
12304 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12305 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12306 Log4Func(("LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12307 break;
12308 }
12309
12310 default:
12311 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12312 VERR_VMX_UNEXPECTED_EXCEPTION);
12313 }
12314
12315 Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS))
12316 == (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS));
12317 if (rcStrict == VINF_IEM_RAISED_XCPT)
12318 {
12319 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
12320 rcStrict = VINF_SUCCESS;
12321 }
12322
12323 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12324 NOREF(pVM);
12325 return rcStrict;
12326}
12327
12328
12329/**
12330 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12331 * VM-exit.
12332 */
12333HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12334{
12335 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12336 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12337
12338 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12339 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12340 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12341 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_EFER);
12342 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12343 AssertRCReturn(rc, rc);
12344
12345 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12346 uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
12347 uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQual);
12348 bool fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12349 bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
12350 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
12351 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12352 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12353
12354 /*
12355 * Update exit history to see if this exit can be optimized.
12356 */
12357 VBOXSTRICTRC rcStrict;
12358 PCEMEXITREC pExitRec = NULL;
12359 if ( !fGstStepping
12360 && !fDbgStepping)
12361 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12362 !fIOString
12363 ? !fIOWrite
12364 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12365 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12366 : !fIOWrite
12367 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12368 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12369 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12370 if (!pExitRec)
12371 {
12372 /* I/O operation lookup arrays. */
12373 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12374 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12375 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12376 uint32_t const cbInstr = pVmxTransient->cbInstr;
12377 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12378 PVM pVM = pVCpu->CTX_SUFF(pVM);
12379 if (fIOString)
12380 {
12381 /*
12382 * INS/OUTS - I/O String instruction.
12383 *
12384 * Use instruction-information if available, otherwise fall back on
12385 * interpreting the instruction.
12386 */
12387 Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12388 AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
12389 bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
12390 if (fInsOutsInfo)
12391 {
12392 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12393 AssertRCReturn(rc2, rc2);
12394 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12395 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12396 IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12397 bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
12398 if (fIOWrite)
12399 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12400 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12401 else
12402 {
12403 /*
12404 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12405 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12406 * See Intel Instruction spec. for "INS".
12407 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12408 */
12409 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12410 }
12411 }
12412 else
12413 rcStrict = IEMExecOne(pVCpu);
12414
12415 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12416 fUpdateRipAlready = true;
12417 }
12418 else
12419 {
12420 /*
12421 * IN/OUT - I/O instruction.
12422 */
12423 Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12424 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12425 Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
12426 if (fIOWrite)
12427 {
12428 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
12429 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12430 if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
12431 && !pCtx->eflags.Bits.u1TF)
12432 rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
12433 }
12434 else
12435 {
12436 uint32_t u32Result = 0;
12437 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12438 if (IOM_SUCCESS(rcStrict))
12439 {
12440 /* Save result of I/O IN instr. in AL/AX/EAX. */
12441 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12442 }
12443 if ( rcStrict == VINF_IOM_R3_IOPORT_READ
12444 && !pCtx->eflags.Bits.u1TF)
12445 rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
12446 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12447 }
12448 }
12449
12450 if (IOM_SUCCESS(rcStrict))
12451 {
12452 if (!fUpdateRipAlready)
12453 {
12454 hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
12455 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12456 }
12457
12458 /*
12459 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12460 * while booting Fedora 17 64-bit guest.
12461 *
12462 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12463 */
12464 if (fIOString)
12465 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12466
12467 /*
12468 * If any I/O breakpoints are armed, we need to check if one triggered
12469 * and take appropriate action.
12470 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12471 */
12472 rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_DR7);
12473 AssertRCReturn(rc, rc);
12474
12475 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12476 * execution engines about whether hyper BPs and such are pending. */
12477 uint32_t const uDr7 = pCtx->dr[7];
12478 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12479 && X86_DR7_ANY_RW_IO(uDr7)
12480 && (pCtx->cr4 & X86_CR4_DE))
12481 || DBGFBpIsHwIoArmed(pVM)))
12482 {
12483 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12484
12485 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12486 VMMRZCallRing3Disable(pVCpu);
12487 HM_DISABLE_PREEMPT(pVCpu);
12488
12489 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12490
12491 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
12492 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12493 {
12494 /* Raise #DB. */
12495 if (fIsGuestDbgActive)
12496 ASMSetDR6(pCtx->dr[6]);
12497 if (pCtx->dr[7] != uDr7)
12498 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR7;
12499
12500 hmR0VmxSetPendingXcptDB(pVCpu);
12501 }
12502 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12503 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12504 else if ( rcStrict2 != VINF_SUCCESS
12505 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12506 rcStrict = rcStrict2;
12507 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12508
12509 HM_RESTORE_PREEMPT();
12510 VMMRZCallRing3Enable(pVCpu);
12511 }
12512 }
12513
12514#ifdef VBOX_STRICT
12515 if ( rcStrict == VINF_IOM_R3_IOPORT_READ
12516 || rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
12517 Assert(!fIOWrite);
12518 else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
12519 || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
12520 || rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
12521 Assert(fIOWrite);
12522 else
12523 {
12524# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12525 * statuses, that the VMM device and some others may return. See
12526 * IOM_SUCCESS() for guidance. */
12527 AssertMsg( RT_FAILURE(rcStrict)
12528 || rcStrict == VINF_SUCCESS
12529 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12530 || rcStrict == VINF_EM_DBG_BREAKPOINT
12531 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12532 || rcStrict == VINF_EM_RAW_TO_R3
12533 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12534# endif
12535 }
12536#endif
12537 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12538 }
12539 else
12540 {
12541 /*
12542 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12543 */
12544 int rc2 = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12545 AssertRCReturn(rc2, rc2);
12546 STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12547 : fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12548 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12549 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12550 VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
12551 fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12552
12553 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12554 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12555
12556 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12557 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12558 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12559 }
12560 return rcStrict;
12561}
12562
12563
12564/**
12565 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12566 * VM-exit.
12567 */
12568HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12569{
12570 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12571
12572 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12573 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12574 AssertRCReturn(rc, rc);
12575 if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12576 {
12577 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12578 AssertRCReturn(rc, rc);
12579 if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
12580 {
12581 uint32_t uErrCode;
12582 RTGCUINTPTR GCPtrFaultAddress;
12583 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12584 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12585 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
12586 if (fErrorCodeValid)
12587 {
12588 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12589 AssertRCReturn(rc, rc);
12590 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12591 }
12592 else
12593 uErrCode = 0;
12594
12595 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12596 && uVector == X86_XCPT_PF)
12597 GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
12598 else
12599 GCPtrFaultAddress = 0;
12600
12601 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12602 AssertRCReturn(rc, rc);
12603
12604 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12605 pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
12606
12607 Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
12608 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12609 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12610 }
12611 }
12612
12613 /* Fall back to the interpreter to emulate the task-switch. */
12614 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12615 return VERR_EM_INTERPRETER;
12616}
12617
12618
12619/**
12620 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12621 */
12622HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12623{
12624 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12625 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_MONITOR_TRAP_FLAG);
12626 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
12627 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12628 AssertRCReturn(rc, rc);
12629 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12630 return VINF_EM_DBG_STEPPED;
12631}
12632
12633
12634/**
12635 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12636 */
12637HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12638{
12639 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12640
12641 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12642
12643 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12644 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12645 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12646 {
12647 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12648 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12649 {
12650 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12651 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12652 }
12653 }
12654 else
12655 {
12656 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12657 rcStrict1 = VINF_SUCCESS;
12658 return rcStrict1;
12659 }
12660
12661 /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12662 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12663 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12664 AssertRCReturn(rc, rc);
12665
12666 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12667 uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
12668 VBOXSTRICTRC rcStrict2;
12669 switch (uAccessType)
12670 {
12671 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12672 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12673 {
12674 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
12675 || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
12676 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12677
12678 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12679 GCPhys &= PAGE_BASE_GC_MASK;
12680 GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
12681 PVM pVM = pVCpu->CTX_SUFF(pVM);
12682 Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12683 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
12684
12685 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12686 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12687 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12688 CPUMCTX2CORE(pCtx), GCPhys);
12689 Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12690 if ( rcStrict2 == VINF_SUCCESS
12691 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12692 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12693 {
12694 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12695 | HM_CHANGED_GUEST_APIC_TPR);
12696 rcStrict2 = VINF_SUCCESS;
12697 }
12698 break;
12699 }
12700
12701 default:
12702 Log4Func(("uAccessType=%#x\n", uAccessType));
12703 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12704 break;
12705 }
12706
12707 if (rcStrict2 != VINF_SUCCESS)
12708 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12709 return rcStrict2;
12710}
12711
12712
12713/**
12714 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12715 * VM-exit.
12716 */
12717HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12718{
12719 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12720
12721 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12722 if (pVmxTransient->fWasGuestDebugStateActive)
12723 {
12724 AssertMsgFailed(("Unexpected MOV DRx exit\n"));
12725 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12726 }
12727
12728 if ( !pVCpu->hm.s.fSingleInstruction
12729 && !pVmxTransient->fWasHyperDebugStateActive)
12730 {
12731 Assert(!DBGFIsStepping(pVCpu));
12732 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12733
12734 /* Don't intercept MOV DRx any more. */
12735 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
12736 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12737 AssertRCReturn(rc, rc);
12738
12739 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12740 VMMRZCallRing3Disable(pVCpu);
12741 HM_DISABLE_PREEMPT(pVCpu);
12742
12743 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12744 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12745 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12746
12747 HM_RESTORE_PREEMPT();
12748 VMMRZCallRing3Enable(pVCpu);
12749
12750#ifdef VBOX_WITH_STATISTICS
12751 rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12752 AssertRCReturn(rc, rc);
12753 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12754 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12755 else
12756 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12757#endif
12758 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12759 return VINF_SUCCESS;
12760 }
12761
12762 /*
12763 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12764 * Update the segment registers and DR7 from the CPU.
12765 */
12766 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12767 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12768 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_DR7);
12769 AssertRCReturn(rc, rc);
12770 Log4Func(("CS:RIP=%04x:%08RX64\n", pCtx->cs.Sel, pCtx->rip));
12771
12772 PVM pVM = pVCpu->CTX_SUFF(pVM);
12773 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12774 {
12775 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12776 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
12777 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
12778 if (RT_SUCCESS(rc))
12779 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12780 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12781 }
12782 else
12783 {
12784 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12785 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
12786 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
12787 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12788 }
12789
12790 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12791 if (RT_SUCCESS(rc))
12792 {
12793 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12794 AssertRCReturn(rc2, rc2);
12795 return VINF_SUCCESS;
12796 }
12797 return rc;
12798}
12799
12800
12801/**
12802 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12803 * Conditional VM-exit.
12804 */
12805HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12806{
12807 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12808 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12809
12810 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12811 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12812 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12813 {
12814 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12815 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12816 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12817 {
12818 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12819 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12820 }
12821 }
12822 else
12823 {
12824 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12825 rcStrict1 = VINF_SUCCESS;
12826 return rcStrict1;
12827 }
12828
12829 /*
12830 * Get sufficent state and update the exit history entry.
12831 */
12832 RTGCPHYS GCPhys;
12833 int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
12834 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12835 AssertRCReturn(rc, rc);
12836
12837 VBOXSTRICTRC rcStrict;
12838 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12839 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
12840 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12841 if (!pExitRec)
12842 {
12843 /*
12844 * If we succeed, resume guest execution.
12845 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12846 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12847 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12848 * weird case. See @bugref{6043}.
12849 */
12850 PVM pVM = pVCpu->CTX_SUFF(pVM);
12851 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12852 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
12853 Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12854 if ( rcStrict == VINF_SUCCESS
12855 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12856 || rcStrict == VERR_PAGE_NOT_PRESENT)
12857 {
12858 /* Successfully handled MMIO operation. */
12859 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12860 | HM_CHANGED_GUEST_APIC_TPR);
12861 rcStrict = VINF_SUCCESS;
12862 }
12863 }
12864 else
12865 {
12866 /*
12867 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12868 */
12869 int rc2 = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12870 AssertRCReturn(rc2, rc2);
12871
12872 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12873 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12874
12875 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12876 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12877
12878 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12879 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12880 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12881 }
12882 return VBOXSTRICTRC_TODO(rcStrict);
12883}
12884
12885
12886/**
12887 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12888 * VM-exit.
12889 */
12890HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12891{
12892 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12893 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12894
12895 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12896 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12897 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12898 {
12899 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12900 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12901 Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12902 }
12903 else
12904 {
12905 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12906 rcStrict1 = VINF_SUCCESS;
12907 return rcStrict1;
12908 }
12909
12910 RTGCPHYS GCPhys;
12911 int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
12912 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12913 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12914 AssertRCReturn(rc, rc);
12915
12916 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12917 AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
12918
12919 RTGCUINT uErrorCode = 0;
12920 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12921 uErrorCode |= X86_TRAP_PF_ID;
12922 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12923 uErrorCode |= X86_TRAP_PF_RW;
12924 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12925 uErrorCode |= X86_TRAP_PF_P;
12926
12927 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12928
12929
12930 /* Handle the pagefault trap for the nested shadow table. */
12931 PVM pVM = pVCpu->CTX_SUFF(pVM);
12932 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12933
12934 Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
12935 pCtx->cs.Sel, pCtx->rip));
12936
12937 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
12938 TRPMResetTrap(pVCpu);
12939
12940 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12941 if ( rcStrict2 == VINF_SUCCESS
12942 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12943 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12944 {
12945 /* Successfully synced our nested page tables. */
12946 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12947 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
12948 return VINF_SUCCESS;
12949 }
12950
12951 Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12952 return rcStrict2;
12953}
12954
12955/** @} */
12956
12957/** @name VM-exit exception handlers.
12958 * @{
12959 */
12960/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12961/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12962/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12963
12964/**
12965 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12966 */
12967static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12968{
12969 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12970 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12971
12972 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0);
12973 AssertRCReturn(rc, rc);
12974
12975 if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
12976 {
12977 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12978 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12979
12980 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12981 * provides VM-exit instruction length. If this causes problem later,
12982 * disassemble the instruction like it's done on AMD-V. */
12983 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12984 AssertRCReturn(rc2, rc2);
12985 return rc;
12986 }
12987
12988 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
12989 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12990 return rc;
12991}
12992
12993
12994/**
12995 * VM-exit exception handler for \#BP (Breakpoint exception).
12996 */
12997static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12998{
12999 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13000 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13001
13002 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13003 AssertRCReturn(rc, rc);
13004
13005 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13006 rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13007 if (rc == VINF_EM_RAW_GUEST_TRAP)
13008 {
13009 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13010 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13011 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13012 AssertRCReturn(rc, rc);
13013
13014 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13015 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13016 }
13017
13018 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
13019 return rc;
13020}
13021
13022
13023/**
13024 * VM-exit exception handler for \#AC (alignment check exception).
13025 */
13026static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13027{
13028 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13029
13030 /*
13031 * Re-inject it. We'll detect any nesting before getting here.
13032 */
13033 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13034 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13035 AssertRCReturn(rc, rc);
13036 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13037
13038 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13039 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13040 return VINF_SUCCESS;
13041}
13042
13043
13044/**
13045 * VM-exit exception handler for \#DB (Debug exception).
13046 */
13047static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13048{
13049 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13050 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13051
13052 /*
13053 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13054 * for processing.
13055 */
13056 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13057
13058 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13059 uint64_t uDR6 = X86_DR6_INIT_VAL;
13060 uDR6 |= (pVmxTransient->uExitQual & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
13061
13062 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13063 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13064 Log6Func(("rc=%Rrc\n", rc));
13065 if (rc == VINF_EM_RAW_GUEST_TRAP)
13066 {
13067 /*
13068 * The exception was for the guest. Update DR6, DR7.GD and
13069 * IA32_DEBUGCTL.LBR before forwarding it.
13070 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13071 */
13072 VMMRZCallRing3Disable(pVCpu);
13073 HM_DISABLE_PREEMPT(pVCpu);
13074
13075 pCtx->dr[6] &= ~X86_DR6_B_MASK;
13076 pCtx->dr[6] |= uDR6;
13077 if (CPUMIsGuestDebugStateActive(pVCpu))
13078 ASMSetDR6(pCtx->dr[6]);
13079
13080 HM_RESTORE_PREEMPT();
13081 VMMRZCallRing3Enable(pVCpu);
13082
13083 rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_DR7);
13084 AssertRCReturn(rc, rc);
13085
13086 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13087 pCtx->dr[7] &= ~X86_DR7_GD;
13088
13089 /* Paranoia. */
13090 pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13091 pCtx->dr[7] |= X86_DR7_RA1_MASK;
13092
13093 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
13094 AssertRCReturn(rc, rc);
13095
13096 /*
13097 * Raise #DB in the guest.
13098 *
13099 * It is important to reflect exactly what the VM-exit gave us (preserving the
13100 * interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
13101 * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
13102 * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
13103 *
13104 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
13105 * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
13106 */
13107 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13108 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13109 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13110 AssertRCReturn(rc, rc);
13111 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13112 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13113 return VINF_SUCCESS;
13114 }
13115
13116 /*
13117 * Not a guest trap, must be a hypervisor related debug event then.
13118 * Update DR6 in case someone is interested in it.
13119 */
13120 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13121 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13122 CPUMSetHyperDR6(pVCpu, uDR6);
13123
13124 return rc;
13125}
13126
13127
13128/**
13129 * Hacks its way around the lovely mesa driver's backdoor accesses.
13130 *
13131 * @sa hmR0SvmHandleMesaDrvGp
13132 */
13133static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
13134{
13135 Log(("hmR0VmxHandleMesaDrvGp: at %04x:%08RX64 rcx=%RX64 rbx=%RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
13136 RT_NOREF(pCtx);
13137
13138 /* For now we'll just skip the instruction. */
13139 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13140}
13141
13142
13143/**
13144 * Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
13145 * backdoor logging w/o checking what it is running inside.
13146 *
13147 * This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
13148 * backdoor port and magic numbers loaded in registers.
13149 *
13150 * @returns true if it is, false if it isn't.
13151 * @sa hmR0SvmIsMesaDrvGp
13152 */
13153DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
13154{
13155 /* 0xed: IN eAX,dx */
13156 uint8_t abInstr[1];
13157 if (pVmxTransient->cbInstr != sizeof(abInstr))
13158 return false;
13159
13160 /* Check that it is #GP(0). */
13161 if (pVmxTransient->uExitIntErrorCode != 0)
13162 return false;
13163
13164 /* Check magic and port. */
13165 Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_RCX)));
13166 /*Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));*/
13167 if (pCtx->rax != UINT32_C(0x564d5868))
13168 return false;
13169 if (pCtx->dx != UINT32_C(0x5658))
13170 return false;
13171
13172 /* Flat ring-3 CS. */
13173 AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
13174 Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
13175 /*Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));*/
13176 if (pCtx->cs.Attr.n.u2Dpl != 3)
13177 return false;
13178 if (pCtx->cs.u64Base != 0)
13179 return false;
13180
13181 /* Check opcode. */
13182 AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
13183 Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
13184 int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
13185 /*Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));*/
13186 if (RT_FAILURE(rc))
13187 return false;
13188 if (abInstr[0] != 0xed)
13189 return false;
13190
13191 return true;
13192}
13193
13194
13195/**
13196 * VM-exit exception handler for \#GP (General-protection exception).
13197 *
13198 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13199 */
13200static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13201{
13202 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13203 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13204
13205 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13206 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13207 { /* likely */ }
13208 else
13209 {
13210#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13211 Assert(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv);
13212#endif
13213 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13214 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13215 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13216 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13217 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13218 AssertRCReturn(rc, rc);
13219 Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pCtx->cs.Sel, pCtx->rip,
13220 pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
13221
13222 if ( !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
13223 || !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
13224 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13225 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13226 else
13227 rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
13228 return rc;
13229 }
13230
13231 Assert(CPUMIsGuestInRealModeEx(pCtx));
13232 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13233
13234 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13235 AssertRCReturn(rc, rc);
13236
13237 VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
13238 if (rcStrict == VINF_SUCCESS)
13239 {
13240 if (!CPUMIsGuestInRealModeEx(pCtx))
13241 {
13242 /*
13243 * The guest is no longer in real-mode, check if we can continue executing the
13244 * guest using hardware-assisted VMX. Otherwise, fall back to emulation.
13245 */
13246 if (HMVmxCanExecuteGuest(pVCpu, pCtx))
13247 {
13248 Log4Func(("Mode changed but guest still suitable for executing using VT-x\n"));
13249 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
13250 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13251 }
13252 else
13253 {
13254 Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
13255 rcStrict = VINF_EM_RESCHEDULE;
13256 }
13257 }
13258 else
13259 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13260 }
13261 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13262 {
13263 rcStrict = VINF_SUCCESS;
13264 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13265 }
13266 return VBOXSTRICTRC_VAL(rcStrict);
13267}
13268
13269
13270/**
13271 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13272 * the exception reported in the VMX transient structure back into the VM.
13273 *
13274 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13275 * up-to-date.
13276 */
13277static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13278{
13279 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13280#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13281 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13282 ("uVector=%#x u32XcptBitmap=%#X32\n",
13283 VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13284#endif
13285
13286 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13287 hmR0VmxCheckExitDueToEventDelivery(). */
13288 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13289 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13290 AssertRCReturn(rc, rc);
13291 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13292
13293#ifdef DEBUG_ramshankar
13294 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
13295 uint8_t uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13296 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13297#endif
13298
13299 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13300 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13301 return VINF_SUCCESS;
13302}
13303
13304
13305/**
13306 * VM-exit exception handler for \#PF (Page-fault exception).
13307 */
13308static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13309{
13310 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13311 PVM pVM = pVCpu->CTX_SUFF(pVM);
13312 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13313 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13314 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13315 AssertRCReturn(rc, rc);
13316
13317 if (!pVM->hm.s.fNestedPaging)
13318 { /* likely */ }
13319 else
13320 {
13321#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13322 Assert(pVCpu->hm.s.fUsingDebugLoop);
13323#endif
13324 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13325 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13326 {
13327 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13328 pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
13329 }
13330 else
13331 {
13332 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13333 hmR0VmxSetPendingXcptDF(pVCpu);
13334 Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13335 }
13336 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13337 return rc;
13338 }
13339
13340 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13341 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13342 if (pVmxTransient->fVectoringPF)
13343 {
13344 Assert(pVCpu->hm.s.Event.fPending);
13345 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13346 }
13347
13348 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13349 rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13350 AssertRCReturn(rc, rc);
13351
13352 Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
13353 pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
13354
13355 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13356 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
13357
13358 Log4Func(("#PF: rc=%Rrc\n", rc));
13359 if (rc == VINF_SUCCESS)
13360 {
13361 /*
13362 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13363 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13364 */
13365 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13366 TRPMResetTrap(pVCpu);
13367 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13368 return rc;
13369 }
13370
13371 if (rc == VINF_EM_RAW_GUEST_TRAP)
13372 {
13373 if (!pVmxTransient->fVectoringDoublePF)
13374 {
13375 /* It's a guest page fault and needs to be reflected to the guest. */
13376 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13377 TRPMResetTrap(pVCpu);
13378 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13379 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13380 uGstErrorCode, pVmxTransient->uExitQual);
13381 }
13382 else
13383 {
13384 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13385 TRPMResetTrap(pVCpu);
13386 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13387 hmR0VmxSetPendingXcptDF(pVCpu);
13388 Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13389 }
13390
13391 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13392 return VINF_SUCCESS;
13393 }
13394
13395 TRPMResetTrap(pVCpu);
13396 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13397 return rc;
13398}
13399
13400/** @} */
13401
13402#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13403/** @name Nested-guest VM-exit handlers.
13404 * @{
13405 */
13406/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13407/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= Nested-guest VM-exit handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13408/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13409
13410/**
13411 * VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
13412 */
13413HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13414{
13415 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13416
13417 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13418 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13419 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13420 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13421 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13422 AssertRCReturn(rc, rc);
13423
13424 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13425
13426 VMXVEXITINFO ExitInfo;
13427 RT_ZERO(ExitInfo);
13428 ExitInfo.uReason = pVmxTransient->uExitReason;
13429 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13430 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13431 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13432 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13433
13434 VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
13435 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13436 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13437 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13438 {
13439 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13440 rcStrict = VINF_SUCCESS;
13441 }
13442 return rcStrict;
13443}
13444
13445
13446/**
13447 * VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
13448 */
13449HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13450{
13451 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13452
13453 /** @todo NSTVMX: Vmlaunch. */
13454 hmR0VmxSetPendingXcptUD(pVCpu);
13455 return VINF_SUCCESS;
13456}
13457
13458
13459/**
13460 * VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
13461 */
13462HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13463{
13464 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13465
13466 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13467 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13468 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13469 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13470 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13471 AssertRCReturn(rc, rc);
13472
13473 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13474
13475 VMXVEXITINFO ExitInfo;
13476 RT_ZERO(ExitInfo);
13477 ExitInfo.uReason = pVmxTransient->uExitReason;
13478 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13479 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13480 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13481 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13482
13483 VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
13484 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13485 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13486 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13487 {
13488 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13489 rcStrict = VINF_SUCCESS;
13490 }
13491 return rcStrict;
13492}
13493
13494
13495/**
13496 * VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
13497 */
13498HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13499{
13500 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13501
13502 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13503 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13504 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13505 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13506 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13507 AssertRCReturn(rc, rc);
13508
13509 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13510
13511 VMXVEXITINFO ExitInfo;
13512 RT_ZERO(ExitInfo);
13513 ExitInfo.uReason = pVmxTransient->uExitReason;
13514 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13515 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13516 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13517 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
13518
13519 VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
13520 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13521 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13522 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13523 {
13524 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13525 rcStrict = VINF_SUCCESS;
13526 }
13527 return rcStrict;
13528}
13529
13530
13531/**
13532 * VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Unconditional VM-exit.
13533 */
13534HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13535{
13536 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13537
13538 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13539 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13540 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13541 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13542 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13543 AssertRCReturn(rc, rc);
13544
13545 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13546
13547 VMXVEXITINFO ExitInfo;
13548 RT_ZERO(ExitInfo);
13549 ExitInfo.uReason = pVmxTransient->uExitReason;
13550 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13551 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13552 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13553 if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
13554 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
13555
13556 VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
13557 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13558 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13559 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13560 {
13561 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13562 rcStrict = VINF_SUCCESS;
13563 }
13564 return rcStrict;
13565}
13566
13567
13568/**
13569 * VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
13570 */
13571HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13572{
13573 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13574
13575 /** @todo NSTVMX: Vmresume. */
13576 hmR0VmxSetPendingXcptUD(pVCpu);
13577 return VINF_SUCCESS;
13578}
13579
13580
13581/**
13582 * VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Unconditional VM-exit.
13583 */
13584HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13585{
13586 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13587
13588 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13589 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13590 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13591 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13592 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13593 AssertRCReturn(rc, rc);
13594
13595 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13596
13597 VMXVEXITINFO ExitInfo;
13598 RT_ZERO(ExitInfo);
13599 ExitInfo.uReason = pVmxTransient->uExitReason;
13600 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13601 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13602 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13603 if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
13604 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13605
13606 VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
13607 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13608 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13609 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13610 {
13611 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13612 rcStrict = VINF_SUCCESS;
13613 }
13614 return rcStrict;
13615}
13616
13617
13618/**
13619 * VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
13620 */
13621HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13622{
13623 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13624
13625 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13626 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13627 AssertRCReturn(rc, rc);
13628
13629 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13630
13631 VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
13632 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13633 {
13634 /* VMXOFF on success changes the internal hwvirt state but not anything that's visible to the guest. */
13635 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
13636 }
13637 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13638 {
13639 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13640 rcStrict = VINF_SUCCESS;
13641 }
13642 return rcStrict;
13643}
13644
13645
13646/**
13647 * VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
13648 */
13649HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13650{
13651 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13652
13653 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13654 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13655 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13656 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13657 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13658 AssertRCReturn(rc, rc);
13659
13660 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13661
13662 VMXVEXITINFO ExitInfo;
13663 RT_ZERO(ExitInfo);
13664 ExitInfo.uReason = pVmxTransient->uExitReason;
13665 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13666 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13667 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13668 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13669
13670 VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
13671 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13672 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13673 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13674 {
13675 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13676 rcStrict = VINF_SUCCESS;
13677 }
13678 return rcStrict;
13679}
13680
13681/** @} */
13682#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13683
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette