VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 75821

Last change on this file since 75821 was 75821, checked in by vboxsync, 6 years ago

HMVMX,ConsoleImpl: Workaround for incorrect assumptions in mesa vmsvga 3d driver. VT-x only.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 578.4 KB
Line 
1/* $Id: HMVMXR0.cpp 75821 2018-11-29 16:43:40Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/x86.h>
25#include <iprt/asm-amd64-x86.h>
26#include <iprt/thread.h>
27
28#include <VBox/vmm/pdmapi.h>
29#include <VBox/vmm/dbgf.h>
30#include <VBox/vmm/iem.h>
31#include <VBox/vmm/iom.h>
32#include <VBox/vmm/selm.h>
33#include <VBox/vmm/tm.h>
34#include <VBox/vmm/em.h>
35#include <VBox/vmm/gim.h>
36#include <VBox/vmm/apic.h>
37#ifdef VBOX_WITH_REM
38# include <VBox/vmm/rem.h>
39#endif
40#include "HMInternal.h"
41#include <VBox/vmm/vm.h>
42#include "HMVMXR0.h"
43#include "dtrace/VBoxVMM.h"
44
45#ifdef DEBUG_ramshankar
46# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
47# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
49# define HMVMX_ALWAYS_CHECK_GUEST_STATE
50# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
51# define HMVMX_ALWAYS_TRAP_PF
52# define HMVMX_ALWAYS_FLUSH_TLB
53# define HMVMX_ALWAYS_SWAP_EFER
54#endif
55
56
57/*********************************************************************************************************************************
58* Defined Constants And Macros *
59*********************************************************************************************************************************/
60/** Use the function table. */
61#define HMVMX_USE_FUNCTION_TABLE
62
63/** Determine which tagged-TLB flush handler to use. */
64#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
65#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
66#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
67#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
68
69/** @name HMVMX_READ_XXX
70 * Flags to skip redundant reads of some common VMCS fields that are not part of
71 * the guest-CPU or VCPU state but are needed while handling VM-exits.
72 */
73#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
74#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
75#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
76#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
77#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
78#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
79#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
80#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
81/** @} */
82
83/**
84 * States of the VMCS.
85 *
86 * This does not reflect all possible VMCS states but currently only those
87 * needed for maintaining the VMCS consistently even when thread-context hooks
88 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
89 */
90#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
91#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
92#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
93
94/**
95 * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
96 * guest using hardware-assisted VMX.
97 *
98 * This excludes state like GPRs (other than RSP) which are always are
99 * swapped and restored across the world-switch and also registers like EFER,
100 * MSR which cannot be modified by the guest without causing a VM-exit.
101 */
102#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
103 | CPUMCTX_EXTRN_RFLAGS \
104 | CPUMCTX_EXTRN_RSP \
105 | CPUMCTX_EXTRN_SREG_MASK \
106 | CPUMCTX_EXTRN_TABLE_MASK \
107 | CPUMCTX_EXTRN_KERNEL_GS_BASE \
108 | CPUMCTX_EXTRN_SYSCALL_MSRS \
109 | CPUMCTX_EXTRN_SYSENTER_MSRS \
110 | CPUMCTX_EXTRN_TSC_AUX \
111 | CPUMCTX_EXTRN_OTHER_MSRS \
112 | CPUMCTX_EXTRN_CR0 \
113 | CPUMCTX_EXTRN_CR3 \
114 | CPUMCTX_EXTRN_CR4 \
115 | CPUMCTX_EXTRN_DR7 \
116 | CPUMCTX_EXTRN_HM_VMX_MASK)
117
118/**
119 * Exception bitmap mask for real-mode guests (real-on-v86).
120 *
121 * We need to intercept all exceptions manually except:
122 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
123 * due to bugs in Intel CPUs.
124 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
125 * support.
126 */
127#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
128 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
129 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
130 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
131 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
132 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
133 | RT_BIT(X86_XCPT_XF))
134
135/** Maximum VM-instruction error number. */
136#define HMVMX_INSTR_ERROR_MAX 28
137
138/** Profiling macro. */
139#ifdef HM_PROFILE_EXIT_DISPATCH
140# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
141# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
142#else
143# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
144# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
145#endif
146
147/** Assert that preemption is disabled or covered by thread-context hooks. */
148#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
149 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
150
151/** Assert that we haven't migrated CPUs when thread-context hooks are not
152 * used. */
153#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
154 || (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
155 ("Illegal migration! Entered on CPU %u Current %u\n", \
156 (a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
157
158/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
159 * context. */
160#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
161 ("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
162 (a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
163
164/** Macro for importing guest state from the VMCS back into CPUMCTX (intended to be
165 * used only from VM-exit handlers). */
166#define HMVMX_CPUMCTX_IMPORT_STATE(a_pVCpu, a_fWhat) (hmR0VmxImportGuestState((a_pVCpu), (a_fWhat)))
167
168/** Helper macro for VM-exit handlers called unexpectedly. */
169#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
170 do { \
171 (a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
172 return VERR_VMX_UNEXPECTED_EXIT; \
173 } while (0)
174
175/** Macro for importing segment registers to the VMCS from the guest-CPU context. */
176#ifdef VMX_USE_CACHED_VMCS_ACCESSES
177# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
178 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
179 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
180#else
181# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
182 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
183 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
184#endif
185
186/** Macro for exporting segment registers to the VMCS from the guest-CPU context. */
187#define HMVMX_EXPORT_SREG(Sel, a_pCtxSelReg) \
188 hmR0VmxExportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
189 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
190
191#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
192/** Macro that does the necessary privilege checks and intercepted VM-exits for
193 * guests that attempted to execute a VMX instruction. */
194# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
195 do \
196 { \
197 VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
198 if (rcStrictTmp == VINF_SUCCESS) \
199 { /* likely */ } \
200 else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
201 { \
202 Assert((a_pVCpu)->hm.s.Event.fPending); \
203 Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
204 return VINF_SUCCESS; \
205 } \
206 else \
207 { \
208 int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
209 AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
210 } \
211 } while (0)
212
213/** Macro that decodes a memory operand for an instruction VM-exit. */
214# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
215 do \
216 { \
217 VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
218 (a_pGCPtrEffAddr)); \
219 if (rcStrictTmp == VINF_SUCCESS) \
220 { /* likely */ } \
221 else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
222 { \
223 uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
224 Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
225 return VINF_SUCCESS; \
226 } \
227 else \
228 { \
229 Log4Func(("hmR0VmxCheckExitDueToVmxInstr failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
230 return rcStrictTmp; \
231 } \
232 } while (0)
233
234#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
235
236
237/*********************************************************************************************************************************
238* Structures and Typedefs *
239*********************************************************************************************************************************/
240/**
241 * VMX transient state.
242 *
243 * A state structure for holding miscellaneous information across
244 * VMX non-root operation and restored after the transition.
245 */
246typedef struct VMXTRANSIENT
247{
248 /** The host's rflags/eflags. */
249 RTCCUINTREG fEFlags;
250#if HC_ARCH_BITS == 32
251 uint32_t u32Alignment0;
252#endif
253 /** The guest's TPR value used for TPR shadowing. */
254 uint8_t u8GuestTpr;
255 /** Alignment. */
256 uint8_t abAlignment0[7];
257
258 /** The basic VM-exit reason. */
259 uint16_t uExitReason;
260 /** Alignment. */
261 uint16_t u16Alignment0;
262 /** The VM-exit interruption error code. */
263 uint32_t uExitIntErrorCode;
264 /** The VM-exit exit code qualification. */
265 uint64_t uExitQual;
266 /** The Guest-linear address. */
267 uint64_t uGuestLinearAddr;
268
269 /** The VM-exit interruption-information field. */
270 uint32_t uExitIntInfo;
271 /** The VM-exit instruction-length field. */
272 uint32_t cbInstr;
273 /** The VM-exit instruction-information field. */
274 VMXEXITINSTRINFO ExitInstrInfo;
275 /** Whether the VM-entry failed or not. */
276 bool fVMEntryFailed;
277 /** Alignment. */
278 uint8_t abAlignment1[3];
279
280 /** The VM-entry interruption-information field. */
281 uint32_t uEntryIntInfo;
282 /** The VM-entry exception error code field. */
283 uint32_t uEntryXcptErrorCode;
284 /** The VM-entry instruction length field. */
285 uint32_t cbEntryInstr;
286
287 /** IDT-vectoring information field. */
288 uint32_t uIdtVectoringInfo;
289 /** IDT-vectoring error code. */
290 uint32_t uIdtVectoringErrorCode;
291
292 /** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
293 uint32_t fVmcsFieldsRead;
294
295 /** Whether the guest debug state was active at the time of VM-exit. */
296 bool fWasGuestDebugStateActive;
297 /** Whether the hyper debug state was active at the time of VM-exit. */
298 bool fWasHyperDebugStateActive;
299 /** Whether TSC-offsetting should be setup before VM-entry. */
300 bool fUpdateTscOffsettingAndPreemptTimer;
301 /** Whether the VM-exit was caused by a page-fault during delivery of a
302 * contributory exception or a page-fault. */
303 bool fVectoringDoublePF;
304 /** Whether the VM-exit was caused by a page-fault during delivery of an
305 * external interrupt or NMI. */
306 bool fVectoringPF;
307} VMXTRANSIENT;
308AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
309AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
310AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
311AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
312AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
313/** Pointer to VMX transient state. */
314typedef VMXTRANSIENT *PVMXTRANSIENT;
315
316/**
317 * Memory operand read or write access.
318 */
319typedef enum VMXMEMACCESS
320{
321 VMXMEMACCESS_READ = 0,
322 VMXMEMACCESS_WRITE = 1
323} VMXMEMACCESS;
324
325/**
326 * VMX VM-exit handler.
327 *
328 * @returns Strict VBox status code (i.e. informational status codes too).
329 * @param pVCpu The cross context virtual CPU structure.
330 * @param pVmxTransient Pointer to the VMX-transient structure.
331 */
332#ifndef HMVMX_USE_FUNCTION_TABLE
333typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
334#else
335typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
336/** Pointer to VM-exit handler. */
337typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
338#endif
339
340/**
341 * VMX VM-exit handler, non-strict status code.
342 *
343 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
344 *
345 * @returns VBox status code, no informational status code returned.
346 * @param pVCpu The cross context virtual CPU structure.
347 * @param pVmxTransient Pointer to the VMX-transient structure.
348 *
349 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
350 * use of that status code will be replaced with VINF_EM_SOMETHING
351 * later when switching over to IEM.
352 */
353#ifndef HMVMX_USE_FUNCTION_TABLE
354typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
355#else
356typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
357#endif
358
359
360/*********************************************************************************************************************************
361* Internal Functions *
362*********************************************************************************************************************************/
363static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush);
364static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr);
365static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
366static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
367static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
368 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
369#if HC_ARCH_BITS == 32
370static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
371#endif
372#ifndef HMVMX_USE_FUNCTION_TABLE
373DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
374# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
375# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
376#else
377# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
378# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
379#endif
380
381/** @name VM-exit handlers.
382 * @{
383 */
384static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
385static FNVMXEXITHANDLER hmR0VmxExitExtInt;
386static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
387static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
388static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
389static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
390static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
391static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
392static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
393static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
394static FNVMXEXITHANDLER hmR0VmxExitCpuid;
395static FNVMXEXITHANDLER hmR0VmxExitGetsec;
396static FNVMXEXITHANDLER hmR0VmxExitHlt;
397static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
398static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
399static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
400static FNVMXEXITHANDLER hmR0VmxExitVmcall;
401#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
402static FNVMXEXITHANDLER hmR0VmxExitVmclear;
403static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
404static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
405static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
406static FNVMXEXITHANDLER hmR0VmxExitVmread;
407static FNVMXEXITHANDLER hmR0VmxExitVmresume;
408static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
409static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
410static FNVMXEXITHANDLER hmR0VmxExitVmxon;
411#endif
412static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
413static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
414static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
415static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
416static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
417static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
418static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
419static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
420static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
421static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
422static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
423static FNVMXEXITHANDLER hmR0VmxExitMwait;
424static FNVMXEXITHANDLER hmR0VmxExitMtf;
425static FNVMXEXITHANDLER hmR0VmxExitMonitor;
426static FNVMXEXITHANDLER hmR0VmxExitPause;
427static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
428static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
429static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
430static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
431static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
432static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
433static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
434static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
435static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
436static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
437static FNVMXEXITHANDLER hmR0VmxExitRdrand;
438static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
439/** @} */
440
441static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
442static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
443static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
444static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
445static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
446static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
447static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
448static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu);
449
450
451/*********************************************************************************************************************************
452* Global Variables *
453*********************************************************************************************************************************/
454#ifdef HMVMX_USE_FUNCTION_TABLE
455
456/**
457 * VMX_EXIT dispatch table.
458 */
459static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
460{
461 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
462 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
463 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
464 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
465 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
466 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
467 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
468 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
469 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
470 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
471 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
472 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
473 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
474 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
475 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
476 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
477 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
478 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
479 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
480#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
481 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
482 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
483 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
484 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
485 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
486 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
487 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
488 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
489 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
490#else
491 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
492 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
493 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
494 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
495 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
496 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
497 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
498 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
499 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
500#endif
501 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
502 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
503 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
504 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
505 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
506 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
507 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
508 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
509 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
510 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
511 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
512 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
513 /* 40 UNDEFINED */ hmR0VmxExitPause,
514 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
515 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
516 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
517 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
518 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
519 /* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitXdtrAccess,
520 /* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitXdtrAccess,
521 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
522 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
523 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
524 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
525 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
526 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
527 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
528 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
529 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
530 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
531 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
532 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
533 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
534 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
535 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
536 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
537 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
538};
539#endif /* HMVMX_USE_FUNCTION_TABLE */
540
541#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
542static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
543{
544 /* 0 */ "(Not Used)",
545 /* 1 */ "VMCALL executed in VMX root operation.",
546 /* 2 */ "VMCLEAR with invalid physical address.",
547 /* 3 */ "VMCLEAR with VMXON pointer.",
548 /* 4 */ "VMLAUNCH with non-clear VMCS.",
549 /* 5 */ "VMRESUME with non-launched VMCS.",
550 /* 6 */ "VMRESUME after VMXOFF",
551 /* 7 */ "VM-entry with invalid control fields.",
552 /* 8 */ "VM-entry with invalid host state fields.",
553 /* 9 */ "VMPTRLD with invalid physical address.",
554 /* 10 */ "VMPTRLD with VMXON pointer.",
555 /* 11 */ "VMPTRLD with incorrect revision identifier.",
556 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
557 /* 13 */ "VMWRITE to read-only VMCS component.",
558 /* 14 */ "(Not Used)",
559 /* 15 */ "VMXON executed in VMX root operation.",
560 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
561 /* 17 */ "VM-entry with non-launched executing VMCS.",
562 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
563 /* 19 */ "VMCALL with non-clear VMCS.",
564 /* 20 */ "VMCALL with invalid VM-exit control fields.",
565 /* 21 */ "(Not Used)",
566 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
567 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
568 /* 24 */ "VMCALL with invalid SMM-monitor features.",
569 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
570 /* 26 */ "VM-entry with events blocked by MOV SS.",
571 /* 27 */ "(Not Used)",
572 /* 28 */ "Invalid operand to INVEPT/INVVPID."
573};
574#endif /* VBOX_STRICT */
575
576
577/**
578 * Updates the VM's last error record.
579 *
580 * If there was a VMX instruction error, reads the error data from the VMCS and
581 * updates VCPU's last error record as well.
582 *
583 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
584 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
585 * VERR_VMX_INVALID_VMCS_FIELD.
586 * @param rc The error code.
587 */
588static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
589{
590 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
591 || rc == VERR_VMX_UNABLE_TO_START_VM)
592 {
593 AssertPtrReturnVoid(pVCpu);
594 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
595 }
596 pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
597}
598
599
600/**
601 * Reads the VM-entry interruption-information field from the VMCS into the VMX
602 * transient structure.
603 *
604 * @returns VBox status code.
605 * @param pVmxTransient Pointer to the VMX transient structure.
606 *
607 * @remarks No-long-jump zone!!!
608 */
609DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
610{
611 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
612 AssertRCReturn(rc, rc);
613 return VINF_SUCCESS;
614}
615
616#ifdef VBOX_STRICT
617/**
618 * Reads the VM-entry exception error code field from the VMCS into
619 * the VMX transient structure.
620 *
621 * @returns VBox status code.
622 * @param pVmxTransient Pointer to the VMX transient structure.
623 *
624 * @remarks No-long-jump zone!!!
625 */
626DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
627{
628 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
629 AssertRCReturn(rc, rc);
630 return VINF_SUCCESS;
631}
632
633
634/**
635 * Reads the VM-entry exception error code field from the VMCS into
636 * the VMX transient structure.
637 *
638 * @returns VBox status code.
639 * @param pVmxTransient Pointer to the VMX transient structure.
640 *
641 * @remarks No-long-jump zone!!!
642 */
643DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
644{
645 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
646 AssertRCReturn(rc, rc);
647 return VINF_SUCCESS;
648}
649#endif /* VBOX_STRICT */
650
651
652/**
653 * Reads the VM-exit interruption-information field from the VMCS into the VMX
654 * transient structure.
655 *
656 * @returns VBox status code.
657 * @param pVmxTransient Pointer to the VMX transient structure.
658 */
659DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
660{
661 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
662 {
663 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
664 AssertRCReturn(rc,rc);
665 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_INFO;
666 }
667 return VINF_SUCCESS;
668}
669
670
671/**
672 * Reads the VM-exit interruption error code from the VMCS into the VMX
673 * transient structure.
674 *
675 * @returns VBox status code.
676 * @param pVmxTransient Pointer to the VMX transient structure.
677 */
678DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
679{
680 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
681 {
682 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
683 AssertRCReturn(rc, rc);
684 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
685 }
686 return VINF_SUCCESS;
687}
688
689
690/**
691 * Reads the VM-exit instruction length field from the VMCS into the VMX
692 * transient structure.
693 *
694 * @returns VBox status code.
695 * @param pVmxTransient Pointer to the VMX transient structure.
696 */
697DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
698{
699 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
700 {
701 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
702 AssertRCReturn(rc, rc);
703 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_LEN;
704 }
705 return VINF_SUCCESS;
706}
707
708
709/**
710 * Reads the VM-exit instruction-information field from the VMCS into
711 * the VMX transient structure.
712 *
713 * @returns VBox status code.
714 * @param pVmxTransient Pointer to the VMX transient structure.
715 */
716DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
717{
718 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
719 {
720 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
721 AssertRCReturn(rc, rc);
722 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_INFO;
723 }
724 return VINF_SUCCESS;
725}
726
727
728/**
729 * Reads the VM-exit Qualification from the VMCS into the VMX transient structure.
730 *
731 * @returns VBox status code.
732 * @param pVCpu The cross context virtual CPU structure of the
733 * calling EMT. (Required for the VMCS cache case.)
734 * @param pVmxTransient Pointer to the VMX transient structure.
735 */
736DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
737{
738 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
739 {
740 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
741 AssertRCReturn(rc, rc);
742 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION;
743 }
744 return VINF_SUCCESS;
745}
746
747
748/**
749 * Reads the Guest-linear address from the VMCS into the VMX transient structure.
750 *
751 * @returns VBox status code.
752 * @param pVCpu The cross context virtual CPU structure of the
753 * calling EMT. (Required for the VMCS cache case.)
754 * @param pVmxTransient Pointer to the VMX transient structure.
755 */
756DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
757{
758 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
759 {
760 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
761 AssertRCReturn(rc, rc);
762 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_GUEST_LINEAR_ADDR;
763 }
764 return VINF_SUCCESS;
765}
766
767
768/**
769 * Reads the IDT-vectoring information field from the VMCS into the VMX
770 * transient structure.
771 *
772 * @returns VBox status code.
773 * @param pVmxTransient Pointer to the VMX transient structure.
774 *
775 * @remarks No-long-jump zone!!!
776 */
777DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
778{
779 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
780 {
781 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
782 AssertRCReturn(rc, rc);
783 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_INFO;
784 }
785 return VINF_SUCCESS;
786}
787
788
789/**
790 * Reads the IDT-vectoring error code from the VMCS into the VMX
791 * transient structure.
792 *
793 * @returns VBox status code.
794 * @param pVmxTransient Pointer to the VMX transient structure.
795 */
796DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
797{
798 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
799 {
800 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
801 AssertRCReturn(rc, rc);
802 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
803 }
804 return VINF_SUCCESS;
805}
806
807
808/**
809 * Enters VMX root mode operation on the current CPU.
810 *
811 * @returns VBox status code.
812 * @param pVM The cross context VM structure. Can be
813 * NULL, after a resume.
814 * @param HCPhysCpuPage Physical address of the VMXON region.
815 * @param pvCpuPage Pointer to the VMXON region.
816 */
817static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
818{
819 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
820 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
821 Assert(pvCpuPage);
822 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
823
824 if (pVM)
825 {
826 /* Write the VMCS revision dword to the VMXON region. */
827 *(uint32_t *)pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
828 }
829
830 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
831 RTCCUINTREG fEFlags = ASMIntDisableFlags();
832
833 /* Enable the VMX bit in CR4 if necessary. */
834 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
835
836 /* Enter VMX root mode. */
837 int rc = VMXEnable(HCPhysCpuPage);
838 if (RT_FAILURE(rc))
839 {
840 if (!(uOldCr4 & X86_CR4_VMXE))
841 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
842
843 if (pVM)
844 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
845 }
846
847 /* Restore interrupts. */
848 ASMSetFlags(fEFlags);
849 return rc;
850}
851
852
853/**
854 * Exits VMX root mode operation on the current CPU.
855 *
856 * @returns VBox status code.
857 */
858static int hmR0VmxLeaveRootMode(void)
859{
860 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
861
862 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
863 RTCCUINTREG fEFlags = ASMIntDisableFlags();
864
865 /* If we're for some reason not in VMX root mode, then don't leave it. */
866 RTCCUINTREG uHostCR4 = ASMGetCR4();
867
868 int rc;
869 if (uHostCR4 & X86_CR4_VMXE)
870 {
871 /* Exit VMX root mode and clear the VMX bit in CR4. */
872 VMXDisable();
873 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
874 rc = VINF_SUCCESS;
875 }
876 else
877 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
878
879 /* Restore interrupts. */
880 ASMSetFlags(fEFlags);
881 return rc;
882}
883
884
885/**
886 * Allocates and maps one physically contiguous page. The allocated page is
887 * zero'd out. (Used by various VT-x structures).
888 *
889 * @returns IPRT status code.
890 * @param pMemObj Pointer to the ring-0 memory object.
891 * @param ppVirt Where to store the virtual address of the
892 * allocation.
893 * @param pHCPhys Where to store the physical address of the
894 * allocation.
895 */
896static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
897{
898 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
899 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
900 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
901
902 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
903 if (RT_FAILURE(rc))
904 return rc;
905 *ppVirt = RTR0MemObjAddress(*pMemObj);
906 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
907 ASMMemZero32(*ppVirt, PAGE_SIZE);
908 return VINF_SUCCESS;
909}
910
911
912/**
913 * Frees and unmaps an allocated physical page.
914 *
915 * @param pMemObj Pointer to the ring-0 memory object.
916 * @param ppVirt Where to re-initialize the virtual address of
917 * allocation as 0.
918 * @param pHCPhys Where to re-initialize the physical address of the
919 * allocation as 0.
920 */
921static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
922{
923 AssertPtr(pMemObj);
924 AssertPtr(ppVirt);
925 AssertPtr(pHCPhys);
926 if (*pMemObj != NIL_RTR0MEMOBJ)
927 {
928 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
929 AssertRC(rc);
930 *pMemObj = NIL_RTR0MEMOBJ;
931 *ppVirt = 0;
932 *pHCPhys = 0;
933 }
934}
935
936
937/**
938 * Worker function to free VT-x related structures.
939 *
940 * @returns IPRT status code.
941 * @param pVM The cross context VM structure.
942 */
943static void hmR0VmxStructsFree(PVM pVM)
944{
945 for (VMCPUID i = 0; i < pVM->cCpus; i++)
946 {
947 PVMCPU pVCpu = &pVM->aCpus[i];
948 AssertPtr(pVCpu);
949
950 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
951 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
952
953 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
954 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
955
956 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
957 }
958
959 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
960#ifdef VBOX_WITH_CRASHDUMP_MAGIC
961 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
962#endif
963}
964
965
966/**
967 * Worker function to allocate VT-x related VM structures.
968 *
969 * @returns IPRT status code.
970 * @param pVM The cross context VM structure.
971 */
972static int hmR0VmxStructsAlloc(PVM pVM)
973{
974 /*
975 * Initialize members up-front so we can cleanup properly on allocation failure.
976 */
977#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
978 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
979 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
980 pVM->hm.s.vmx.HCPhys##a_Name = 0;
981
982#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
983 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
984 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
985 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
986
987#ifdef VBOX_WITH_CRASHDUMP_MAGIC
988 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
989#endif
990 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
991
992 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
993 for (VMCPUID i = 0; i < pVM->cCpus; i++)
994 {
995 PVMCPU pVCpu = &pVM->aCpus[i];
996 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
997 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
998 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
999 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
1000 }
1001#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
1002#undef VMXLOCAL_INIT_VM_MEMOBJ
1003
1004 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
1005 AssertReturnStmt(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE) <= PAGE_SIZE,
1006 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
1007 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
1008
1009 /*
1010 * Allocate all the VT-x structures.
1011 */
1012 int rc = VINF_SUCCESS;
1013#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1014 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1015 if (RT_FAILURE(rc))
1016 goto cleanup;
1017 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1018 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1019#endif
1020
1021 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1022 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1023 {
1024 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1025 &pVM->hm.s.vmx.HCPhysApicAccess);
1026 if (RT_FAILURE(rc))
1027 goto cleanup;
1028 }
1029
1030 /*
1031 * Initialize per-VCPU VT-x structures.
1032 */
1033 for (VMCPUID i = 0; i < pVM->cCpus; i++)
1034 {
1035 PVMCPU pVCpu = &pVM->aCpus[i];
1036 AssertPtr(pVCpu);
1037
1038 /* Allocate the VM control structure (VMCS). */
1039 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1040 if (RT_FAILURE(rc))
1041 goto cleanup;
1042
1043 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1044 if ( PDMHasApic(pVM)
1045 && (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1046 {
1047 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1048 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1049 if (RT_FAILURE(rc))
1050 goto cleanup;
1051 }
1052
1053 /*
1054 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1055 * transparent accesses of specific MSRs.
1056 *
1057 * If the condition for enabling MSR bitmaps changes here, don't forget to
1058 * update HMAreMsrBitmapsAvailable().
1059 */
1060 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1061 {
1062 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1063 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1064 if (RT_FAILURE(rc))
1065 goto cleanup;
1066 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1067 }
1068
1069 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1070 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1071 if (RT_FAILURE(rc))
1072 goto cleanup;
1073
1074 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1075 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1076 if (RT_FAILURE(rc))
1077 goto cleanup;
1078 }
1079
1080 return VINF_SUCCESS;
1081
1082cleanup:
1083 hmR0VmxStructsFree(pVM);
1084 return rc;
1085}
1086
1087
1088/**
1089 * Does global VT-x initialization (called during module initialization).
1090 *
1091 * @returns VBox status code.
1092 */
1093VMMR0DECL(int) VMXR0GlobalInit(void)
1094{
1095#ifdef HMVMX_USE_FUNCTION_TABLE
1096 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1097# ifdef VBOX_STRICT
1098 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1099 Assert(g_apfnVMExitHandlers[i]);
1100# endif
1101#endif
1102 return VINF_SUCCESS;
1103}
1104
1105
1106/**
1107 * Does global VT-x termination (called during module termination).
1108 */
1109VMMR0DECL(void) VMXR0GlobalTerm()
1110{
1111 /* Nothing to do currently. */
1112}
1113
1114
1115/**
1116 * Sets up and activates VT-x on the current CPU.
1117 *
1118 * @returns VBox status code.
1119 * @param pHostCpu Pointer to the global CPU info struct.
1120 * @param pVM The cross context VM structure. Can be
1121 * NULL after a host resume operation.
1122 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1123 * fEnabledByHost is @c true).
1124 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1125 * @a fEnabledByHost is @c true).
1126 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1127 * enable VT-x on the host.
1128 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1129 */
1130VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1131 void *pvMsrs)
1132{
1133 Assert(pHostCpu);
1134 Assert(pvMsrs);
1135 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1136
1137 /* Enable VT-x if it's not already enabled by the host. */
1138 if (!fEnabledByHost)
1139 {
1140 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1141 if (RT_FAILURE(rc))
1142 return rc;
1143 }
1144
1145 /*
1146 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1147 * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1148 * invalidated when flushing by VPID.
1149 */
1150 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1151 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1152 {
1153 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
1154 pHostCpu->fFlushAsidBeforeUse = false;
1155 }
1156 else
1157 pHostCpu->fFlushAsidBeforeUse = true;
1158
1159 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1160 ++pHostCpu->cTlbFlushes;
1161
1162 return VINF_SUCCESS;
1163}
1164
1165
1166/**
1167 * Deactivates VT-x on the current CPU.
1168 *
1169 * @returns VBox status code.
1170 * @param pHostCpu Pointer to the global CPU info struct.
1171 * @param pvCpuPage Pointer to the VMXON region.
1172 * @param HCPhysCpuPage Physical address of the VMXON region.
1173 *
1174 * @remarks This function should never be called when SUPR0EnableVTx() or
1175 * similar was used to enable VT-x on the host.
1176 */
1177VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1178{
1179 RT_NOREF3(pHostCpu, pvCpuPage, HCPhysCpuPage);
1180
1181 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1182 return hmR0VmxLeaveRootMode();
1183}
1184
1185
1186/**
1187 * Sets the permission bits for the specified MSR in the MSR bitmap.
1188 *
1189 * @param pVCpu The cross context virtual CPU structure.
1190 * @param uMsr The MSR value.
1191 * @param enmRead Whether reading this MSR causes a VM-exit.
1192 * @param enmWrite Whether writing this MSR causes a VM-exit.
1193 */
1194static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1195{
1196 int32_t iBit;
1197 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1198
1199 /*
1200 * MSR Layout:
1201 * Byte index MSR range Interpreted as
1202 * 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
1203 * 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
1204 * 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
1205 * 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
1206 *
1207 * A bit corresponding to an MSR within the above range causes a VM-exit
1208 * if the bit is 1 on executions of RDMSR/WRMSR.
1209 *
1210 * If an MSR falls out of the MSR range, it always cause a VM-exit.
1211 *
1212 * See Intel spec. 24.6.9 "MSR-Bitmap Address".
1213 */
1214 if (uMsr <= 0x00001fff)
1215 iBit = uMsr;
1216 else if (uMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1217 {
1218 iBit = uMsr - UINT32_C(0xc0000000);
1219 pbMsrBitmap += 0x400;
1220 }
1221 else
1222 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1223
1224 Assert(iBit <= 0x1fff);
1225 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1226 ASMBitSet(pbMsrBitmap, iBit);
1227 else
1228 ASMBitClear(pbMsrBitmap, iBit);
1229
1230 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1231 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1232 else
1233 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1234}
1235
1236
1237/**
1238 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1239 * area.
1240 *
1241 * @returns VBox status code.
1242 * @param pVCpu The cross context virtual CPU structure.
1243 * @param cMsrs The number of MSRs.
1244 */
1245static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1246{
1247 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1248 uint64_t const uVmxMiscMsr = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc;
1249 uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(uVmxMiscMsr);
1250 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1251 {
1252 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1253 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1254 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1255 }
1256
1257 /* Update number of guest MSRs to load/store across the world-switch. */
1258 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1259 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1260
1261 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1262 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1263 AssertRCReturn(rc, rc);
1264
1265 /* Update the VCPU's copy of the MSR count. */
1266 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1267
1268 return VINF_SUCCESS;
1269}
1270
1271
1272/**
1273 * Adds a new (or updates the value of an existing) guest/host MSR
1274 * pair to be swapped during the world-switch as part of the
1275 * auto-load/store MSR area in the VMCS.
1276 *
1277 * @returns VBox status code.
1278 * @param pVCpu The cross context virtual CPU structure.
1279 * @param uMsr The MSR.
1280 * @param uGuestMsrValue Value of the guest MSR.
1281 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1282 * necessary.
1283 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1284 * its value was updated. Optional, can be NULL.
1285 */
1286static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1287 bool *pfAddedAndUpdated)
1288{
1289 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1290 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1291 uint32_t i;
1292 for (i = 0; i < cMsrs; i++)
1293 {
1294 if (pGuestMsr->u32Msr == uMsr)
1295 break;
1296 pGuestMsr++;
1297 }
1298
1299 bool fAdded = false;
1300 if (i == cMsrs)
1301 {
1302 ++cMsrs;
1303 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1304 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1305
1306 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1307 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1308 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1309
1310 fAdded = true;
1311 }
1312
1313 /* Update the MSR values in the auto-load/store MSR area. */
1314 pGuestMsr->u32Msr = uMsr;
1315 pGuestMsr->u64Value = uGuestMsrValue;
1316
1317 /* Create/update the MSR slot in the host MSR area. */
1318 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1319 pHostMsr += i;
1320 pHostMsr->u32Msr = uMsr;
1321
1322 /*
1323 * Update the host MSR only when requested by the caller AND when we're
1324 * adding it to the auto-load/store area. Otherwise, it would have been
1325 * updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1326 */
1327 bool fUpdatedMsrValue = false;
1328 if ( fAdded
1329 && fUpdateHostMsr)
1330 {
1331 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1332 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1333 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1334 fUpdatedMsrValue = true;
1335 }
1336
1337 if (pfAddedAndUpdated)
1338 *pfAddedAndUpdated = fUpdatedMsrValue;
1339 return VINF_SUCCESS;
1340}
1341
1342
1343/**
1344 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1345 * auto-load/store MSR area in the VMCS.
1346 *
1347 * @returns VBox status code.
1348 * @param pVCpu The cross context virtual CPU structure.
1349 * @param uMsr The MSR.
1350 */
1351static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1352{
1353 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1354 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1355 for (uint32_t i = 0; i < cMsrs; i++)
1356 {
1357 /* Find the MSR. */
1358 if (pGuestMsr->u32Msr == uMsr)
1359 {
1360 /* If it's the last MSR, simply reduce the count. */
1361 if (i == cMsrs - 1)
1362 {
1363 --cMsrs;
1364 break;
1365 }
1366
1367 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1368 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1369 pLastGuestMsr += cMsrs - 1;
1370 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1371 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1372
1373 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1374 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1375 pLastHostMsr += cMsrs - 1;
1376 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1377 pHostMsr->u64Value = pLastHostMsr->u64Value;
1378 --cMsrs;
1379 break;
1380 }
1381 pGuestMsr++;
1382 }
1383
1384 /* Update the VMCS if the count changed (meaning the MSR was found). */
1385 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1386 {
1387 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1388 AssertRCReturn(rc, rc);
1389
1390 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1391 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1392 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1393
1394 Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1395 return VINF_SUCCESS;
1396 }
1397
1398 return VERR_NOT_FOUND;
1399}
1400
1401
1402/**
1403 * Checks if the specified guest MSR is part of the auto-load/store area in
1404 * the VMCS.
1405 *
1406 * @returns true if found, false otherwise.
1407 * @param pVCpu The cross context virtual CPU structure.
1408 * @param uMsr The MSR to find.
1409 */
1410static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1411{
1412 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1413 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1414
1415 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1416 {
1417 if (pGuestMsr->u32Msr == uMsr)
1418 return true;
1419 }
1420 return false;
1421}
1422
1423
1424/**
1425 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1426 *
1427 * @param pVCpu The cross context virtual CPU structure.
1428 *
1429 * @remarks No-long-jump zone!!!
1430 */
1431static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1432{
1433 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1434 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1435 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1436 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1437
1438 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1439 {
1440 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1441
1442 /*
1443 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1444 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1445 */
1446 if (pHostMsr->u32Msr == MSR_K6_EFER)
1447 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1448 else
1449 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1450 }
1451
1452 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1453}
1454
1455
1456/**
1457 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1458 * perform lazy restoration of the host MSRs while leaving VT-x.
1459 *
1460 * @param pVCpu The cross context virtual CPU structure.
1461 *
1462 * @remarks No-long-jump zone!!!
1463 */
1464static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1465{
1466 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1467
1468 /*
1469 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1470 */
1471 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1472 {
1473 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1474#if HC_ARCH_BITS == 64
1475 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1476 {
1477 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1478 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1479 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1480 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1481 }
1482#endif
1483 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1484 }
1485}
1486
1487
1488/**
1489 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1490 * lazily while leaving VT-x.
1491 *
1492 * @returns true if it does, false otherwise.
1493 * @param pVCpu The cross context virtual CPU structure.
1494 * @param uMsr The MSR to check.
1495 */
1496static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1497{
1498 NOREF(pVCpu);
1499#if HC_ARCH_BITS == 64
1500 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1501 {
1502 switch (uMsr)
1503 {
1504 case MSR_K8_LSTAR:
1505 case MSR_K6_STAR:
1506 case MSR_K8_SF_MASK:
1507 case MSR_K8_KERNEL_GS_BASE:
1508 return true;
1509 }
1510 }
1511#else
1512 RT_NOREF(pVCpu, uMsr);
1513#endif
1514 return false;
1515}
1516
1517
1518/**
1519 * Loads a set of guests MSRs to allow read/passthru to the guest.
1520 *
1521 * The name of this function is slightly confusing. This function does NOT
1522 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1523 * common prefix for functions dealing with "lazy restoration" of the shared
1524 * MSRs.
1525 *
1526 * @param pVCpu The cross context virtual CPU structure.
1527 *
1528 * @remarks No-long-jump zone!!!
1529 */
1530static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1531{
1532 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1533 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1534
1535 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1536#if HC_ARCH_BITS == 64
1537 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1538 {
1539 /*
1540 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1541 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1542 * we can skip a few MSR writes.
1543 *
1544 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1545 * guest MSR values in the guest-CPU context might be different to what's currently
1546 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1547 * CPU, see @bugref{8728}.
1548 */
1549 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1550 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1551 && pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1552 && pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1553 && pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1554 && pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1555 {
1556#ifdef VBOX_STRICT
1557 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1558 Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1559 Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1560 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
1561#endif
1562 }
1563 else
1564 {
1565 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1566 ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
1567 ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
1568 ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
1569 }
1570 }
1571#endif
1572 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1573}
1574
1575
1576/**
1577 * Performs lazy restoration of the set of host MSRs if they were previously
1578 * loaded with guest MSR values.
1579 *
1580 * @param pVCpu The cross context virtual CPU structure.
1581 *
1582 * @remarks No-long-jump zone!!!
1583 * @remarks The guest MSRs should have been saved back into the guest-CPU
1584 * context by hmR0VmxImportGuestState()!!!
1585 */
1586static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1587{
1588 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1589 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1590
1591 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1592 {
1593 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1594#if HC_ARCH_BITS == 64
1595 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1596 {
1597 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1598 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1599 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1600 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1601 }
1602#endif
1603 }
1604 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1605}
1606
1607
1608/**
1609 * Verifies that our cached values of the VMCS fields are all consistent with
1610 * what's actually present in the VMCS.
1611 *
1612 * @returns VBox status code.
1613 * @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
1614 * @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
1615 * VMCS content. HMCPU error-field is
1616 * updated, see VMX_VCI_XXX.
1617 * @param pVCpu The cross context virtual CPU structure.
1618 */
1619static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1620{
1621 uint32_t u32Val;
1622 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1623 AssertRCReturn(rc, rc);
1624 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32EntryCtls == u32Val,
1625 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1626 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
1627 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1628
1629 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1630 AssertRCReturn(rc, rc);
1631 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ExitCtls == u32Val,
1632 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1633 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
1634 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1635
1636 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1637 AssertRCReturn(rc, rc);
1638 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32PinCtls == u32Val,
1639 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1640 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
1641 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1642
1643 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1644 AssertRCReturn(rc, rc);
1645 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls == u32Val,
1646 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1647 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
1648 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1649
1650 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
1651 {
1652 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1653 AssertRCReturn(rc, rc);
1654 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1655 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1656 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
1657 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1658 }
1659
1660 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
1661 AssertRCReturn(rc, rc);
1662 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32XcptBitmap == u32Val,
1663 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap, u32Val),
1664 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
1665 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1666
1667 uint64_t u64Val;
1668 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
1669 AssertRCReturn(rc, rc);
1670 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u64TscOffset == u64Val,
1671 ("Cache=%#RX64 VMCS=%#RX64\n", pVCpu->hm.s.vmx.u64TscOffset, u64Val),
1672 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
1673 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1674
1675 return VINF_SUCCESS;
1676}
1677
1678
1679#ifdef VBOX_STRICT
1680/**
1681 * Verifies that our cached host EFER value has not changed
1682 * since we cached it.
1683 *
1684 * @param pVCpu The cross context virtual CPU structure.
1685 */
1686static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1687{
1688 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1689
1690 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
1691 {
1692 uint64_t u64Val;
1693 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1694 AssertRC(rc);
1695
1696 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1697 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1698 }
1699}
1700
1701
1702/**
1703 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1704 * VMCS are correct.
1705 *
1706 * @param pVCpu The cross context virtual CPU structure.
1707 */
1708static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1709{
1710 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1711
1712 /* Verify MSR counts in the VMCS are what we think it should be. */
1713 uint32_t cMsrs;
1714 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1715 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1716
1717 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1718 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1719
1720 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1721 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1722
1723 PCVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1724 PCVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1725 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1726 {
1727 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1728 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1729 pGuestMsr->u32Msr, cMsrs));
1730
1731 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1732 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1733 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1734
1735 /* Verify that the permissions are as expected in the MSR bitmap. */
1736 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1737 {
1738 VMXMSREXITREAD enmRead;
1739 VMXMSREXITWRITE enmWrite;
1740 rc = HMVmxGetMsrPermission(pVCpu->hm.s.vmx.pvMsrBitmap, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1741 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("HMVmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1742 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1743 {
1744 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1745 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1746 }
1747 else
1748 {
1749 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1750 pGuestMsr->u32Msr, cMsrs));
1751 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1752 pGuestMsr->u32Msr, cMsrs));
1753 }
1754 }
1755 }
1756}
1757#endif /* VBOX_STRICT */
1758
1759
1760/**
1761 * Flushes the TLB using EPT.
1762 *
1763 * @returns VBox status code.
1764 * @param pVCpu The cross context virtual CPU structure of the calling
1765 * EMT. Can be NULL depending on @a enmTlbFlush.
1766 * @param enmTlbFlush Type of flush.
1767 *
1768 * @remarks Caller is responsible for making sure this function is called only
1769 * when NestedPaging is supported and providing @a enmTlbFlush that is
1770 * supported by the CPU.
1771 * @remarks Can be called with interrupts disabled.
1772 */
1773static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush)
1774{
1775 uint64_t au64Descriptor[2];
1776 if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
1777 au64Descriptor[0] = 0;
1778 else
1779 {
1780 Assert(pVCpu);
1781 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1782 }
1783 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1784
1785 int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
1786 AssertMsg(rc == VINF_SUCCESS,
1787 ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0, rc));
1788
1789 if ( RT_SUCCESS(rc)
1790 && pVCpu)
1791 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1792}
1793
1794
1795/**
1796 * Flushes the TLB using VPID.
1797 *
1798 * @returns VBox status code.
1799 * @param pVCpu The cross context virtual CPU structure of the calling
1800 * EMT. Can be NULL depending on @a enmTlbFlush.
1801 * @param enmTlbFlush Type of flush.
1802 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1803 * on @a enmTlbFlush).
1804 *
1805 * @remarks Can be called with interrupts disabled.
1806 */
1807static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
1808{
1809 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
1810
1811 uint64_t au64Descriptor[2];
1812 if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
1813 {
1814 au64Descriptor[0] = 0;
1815 au64Descriptor[1] = 0;
1816 }
1817 else
1818 {
1819 AssertPtr(pVCpu);
1820 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1821 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1822 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1823 au64Descriptor[1] = GCPtr;
1824 }
1825
1826 int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
1827 AssertMsg(rc == VINF_SUCCESS,
1828 ("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1829
1830 if ( RT_SUCCESS(rc)
1831 && pVCpu)
1832 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1833 NOREF(rc);
1834}
1835
1836
1837/**
1838 * Invalidates a guest page by guest virtual address. Only relevant for
1839 * EPT/VPID, otherwise there is nothing really to invalidate.
1840 *
1841 * @returns VBox status code.
1842 * @param pVCpu The cross context virtual CPU structure.
1843 * @param GCVirt Guest virtual address of the page to invalidate.
1844 */
1845VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
1846{
1847 AssertPtr(pVCpu);
1848 LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
1849
1850 bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1851 if (!fFlushPending)
1852 {
1853 /*
1854 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1855 * the EPT case. See @bugref{6043} and @bugref{6177}.
1856 *
1857 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1858 * as this function maybe called in a loop with individual addresses.
1859 */
1860 PVM pVM = pVCpu->CTX_SUFF(pVM);
1861 if (pVM->hm.s.vmx.fVpid)
1862 {
1863 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1864
1865#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1866 /*
1867 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1868 * where executing INVVPID outside 64-bit mode does not flush translations of
1869 * 64-bit linear addresses, see @bugref{6208#c72}.
1870 */
1871 if (RT_HI_U32(GCVirt))
1872 fVpidFlush = false;
1873#endif
1874
1875 if (fVpidFlush)
1876 {
1877 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
1878 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1879 }
1880 else
1881 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1882 }
1883 else if (pVM->hm.s.fNestedPaging)
1884 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1885 }
1886
1887 return VINF_SUCCESS;
1888}
1889
1890
1891/**
1892 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1893 * case where neither EPT nor VPID is supported by the CPU.
1894 *
1895 * @param pVCpu The cross context virtual CPU structure.
1896 * @param pCpu Pointer to the global HM struct.
1897 *
1898 * @remarks Called with interrupts disabled.
1899 */
1900static void hmR0VmxFlushTaggedTlbNone(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1901{
1902 AssertPtr(pVCpu);
1903 AssertPtr(pCpu);
1904
1905 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1906
1907 Assert(pCpu->idCpu != NIL_RTCPUID);
1908 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1909 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1910 pVCpu->hm.s.fForceTLBFlush = false;
1911 return;
1912}
1913
1914
1915/**
1916 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1917 *
1918 * @param pVCpu The cross context virtual CPU structure.
1919 * @param pCpu Pointer to the global HM CPU struct.
1920 *
1921 * @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
1922 * nomenclature. The reason is, to avoid confusion in compare statements
1923 * since the host-CPU copies are named "ASID".
1924 *
1925 * @remarks Called with interrupts disabled.
1926 */
1927static void hmR0VmxFlushTaggedTlbBoth(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1928{
1929#ifdef VBOX_WITH_STATISTICS
1930 bool fTlbFlushed = false;
1931# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1932# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1933 if (!fTlbFlushed) \
1934 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1935 } while (0)
1936#else
1937# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1938# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1939#endif
1940
1941 AssertPtr(pCpu);
1942 AssertPtr(pVCpu);
1943 Assert(pCpu->idCpu != NIL_RTCPUID);
1944
1945 PVM pVM = pVCpu->CTX_SUFF(pVM);
1946 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1947 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1948 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1949
1950 /*
1951 * Force a TLB flush for the first world-switch if the current CPU differs from the one we
1952 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1953 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1954 * cannot reuse the current ASID anymore.
1955 */
1956 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1957 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1958 {
1959 ++pCpu->uCurrentAsid;
1960 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1961 {
1962 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1963 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1964 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1965 }
1966
1967 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1968 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1969 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1970
1971 /*
1972 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1973 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1974 */
1975 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1976 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1977 HMVMX_SET_TAGGED_TLB_FLUSHED();
1978 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1979 }
1980 else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
1981 {
1982 /*
1983 * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
1984 * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
1985 * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
1986 * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
1987 * mappings, see @bugref{6568}.
1988 *
1989 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
1990 */
1991 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1992 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1993 HMVMX_SET_TAGGED_TLB_FLUSHED();
1994 }
1995
1996 pVCpu->hm.s.fForceTLBFlush = false;
1997 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1998
1999 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2000 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2001 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2002 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2003 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2004 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2005 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2006 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2007 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2008
2009 /* Update VMCS with the VPID. */
2010 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2011 AssertRC(rc);
2012
2013#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2014}
2015
2016
2017/**
2018 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2019 *
2020 * @returns VBox status code.
2021 * @param pVCpu The cross context virtual CPU structure.
2022 * @param pCpu Pointer to the global HM CPU struct.
2023 *
2024 * @remarks Called with interrupts disabled.
2025 */
2026static void hmR0VmxFlushTaggedTlbEpt(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2027{
2028 AssertPtr(pVCpu);
2029 AssertPtr(pCpu);
2030 Assert(pCpu->idCpu != NIL_RTCPUID);
2031 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2032 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2033
2034 /*
2035 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2036 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2037 */
2038 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2039 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2040 {
2041 pVCpu->hm.s.fForceTLBFlush = true;
2042 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2043 }
2044
2045 /* Check for explicit TLB flushes. */
2046 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2047 {
2048 pVCpu->hm.s.fForceTLBFlush = true;
2049 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2050 }
2051
2052 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2053 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2054
2055 if (pVCpu->hm.s.fForceTLBFlush)
2056 {
2057 hmR0VmxFlushEpt(pVCpu, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2058 pVCpu->hm.s.fForceTLBFlush = false;
2059 }
2060}
2061
2062
2063/**
2064 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2065 *
2066 * @returns VBox status code.
2067 * @param pVCpu The cross context virtual CPU structure.
2068 * @param pCpu Pointer to the global HM CPU struct.
2069 *
2070 * @remarks Called with interrupts disabled.
2071 */
2072static void hmR0VmxFlushTaggedTlbVpid(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2073{
2074 AssertPtr(pVCpu);
2075 AssertPtr(pCpu);
2076 Assert(pCpu->idCpu != NIL_RTCPUID);
2077 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2078 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2079
2080 /*
2081 * Force a TLB flush for the first world switch if the current CPU differs from the one we
2082 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2083 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2084 * cannot reuse the current ASID anymore.
2085 */
2086 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2087 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2088 {
2089 pVCpu->hm.s.fForceTLBFlush = true;
2090 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2091 }
2092
2093 /* Check for explicit TLB flushes. */
2094 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2095 {
2096 /*
2097 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2098 * hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2099 * fExplicitFlush = true here and change the pCpu->fFlushAsidBeforeUse check below to
2100 * include fExplicitFlush's too) - an obscure corner case.
2101 */
2102 pVCpu->hm.s.fForceTLBFlush = true;
2103 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2104 }
2105
2106 PVM pVM = pVCpu->CTX_SUFF(pVM);
2107 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2108 if (pVCpu->hm.s.fForceTLBFlush)
2109 {
2110 ++pCpu->uCurrentAsid;
2111 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2112 {
2113 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2114 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2115 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2116 }
2117
2118 pVCpu->hm.s.fForceTLBFlush = false;
2119 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2120 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2121 if (pCpu->fFlushAsidBeforeUse)
2122 {
2123 if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2124 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2125 else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2126 {
2127 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2128 pCpu->fFlushAsidBeforeUse = false;
2129 }
2130 else
2131 {
2132 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2133 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2134 }
2135 }
2136 }
2137
2138 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2139 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2140 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2141 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2142 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2143 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2144 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2145
2146 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2147 AssertRC(rc);
2148}
2149
2150
2151/**
2152 * Flushes the guest TLB entry based on CPU capabilities.
2153 *
2154 * @param pVCpu The cross context virtual CPU structure.
2155 * @param pCpu Pointer to the global HM CPU struct.
2156 */
2157DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2158{
2159#ifdef HMVMX_ALWAYS_FLUSH_TLB
2160 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2161#endif
2162 PVM pVM = pVCpu->CTX_SUFF(pVM);
2163 switch (pVM->hm.s.vmx.enmTlbFlushType)
2164 {
2165 case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVCpu, pCpu); break;
2166 case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pVCpu, pCpu); break;
2167 case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pVCpu, pCpu); break;
2168 case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pVCpu, pCpu); break;
2169 default:
2170 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2171 break;
2172 }
2173 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2174}
2175
2176
2177/**
2178 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2179 * TLB entries from the host TLB before VM-entry.
2180 *
2181 * @returns VBox status code.
2182 * @param pVM The cross context VM structure.
2183 */
2184static int hmR0VmxSetupTaggedTlb(PVM pVM)
2185{
2186 /*
2187 * Determine optimal flush type for Nested Paging.
2188 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2189 * guest execution (see hmR3InitFinalizeR0()).
2190 */
2191 if (pVM->hm.s.fNestedPaging)
2192 {
2193 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2194 {
2195 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2196 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2197 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2198 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2199 else
2200 {
2201 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2202 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2203 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2204 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2205 }
2206
2207 /* Make sure the write-back cacheable memory type for EPT is supported. */
2208 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2209 {
2210 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2211 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2212 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2213 }
2214
2215 /* EPT requires a page-walk length of 4. */
2216 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2217 {
2218 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2219 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2220 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2221 }
2222 }
2223 else
2224 {
2225 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2226 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2227 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2228 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2229 }
2230 }
2231
2232 /*
2233 * Determine optimal flush type for VPID.
2234 */
2235 if (pVM->hm.s.vmx.fVpid)
2236 {
2237 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2238 {
2239 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2240 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2241 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2242 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2243 else
2244 {
2245 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2246 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2247 LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2248 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2249 LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2250 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2251 pVM->hm.s.vmx.fVpid = false;
2252 }
2253 }
2254 else
2255 {
2256 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2257 Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2258 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2259 pVM->hm.s.vmx.fVpid = false;
2260 }
2261 }
2262
2263 /*
2264 * Setup the handler for flushing tagged-TLBs.
2265 */
2266 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2267 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2268 else if (pVM->hm.s.fNestedPaging)
2269 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2270 else if (pVM->hm.s.vmx.fVpid)
2271 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2272 else
2273 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2274 return VINF_SUCCESS;
2275}
2276
2277
2278/**
2279 * Sets up pin-based VM-execution controls in the VMCS.
2280 *
2281 * @returns VBox status code.
2282 * @param pVCpu The cross context virtual CPU structure.
2283 *
2284 * @remarks We don't really care about optimizing vmwrites here as it's done only
2285 * once per VM and hence we don't care about VMCS-field cache comparisons.
2286 */
2287static int hmR0VmxSetupPinCtls(PVMCPU pVCpu)
2288{
2289 PVM pVM = pVCpu->CTX_SUFF(pVM);
2290 uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.disallowed0; /* Bits set here must always be set. */
2291 uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2292
2293 fVal |= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2294 | VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2295
2296 if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2297 fVal |= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2298
2299 /* Enable the VMX preemption timer. */
2300 if (pVM->hm.s.vmx.fUsePreemptTimer)
2301 {
2302 Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2303 fVal |= VMX_PIN_CTLS_PREEMPT_TIMER;
2304 }
2305
2306#if 0
2307 /* Enable posted-interrupt processing. */
2308 if (pVM->hm.s.fPostedIntrs)
2309 {
2310 Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2311 Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2312 fVal |= VMX_PIN_CTL_POSTED_INT;
2313 }
2314#endif
2315
2316 if ((fVal & fZap) != fVal)
2317 {
2318 LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2319 pVM->hm.s.vmx.Msrs.PinCtls.n.disallowed0, fVal, fZap));
2320 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2321 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2322 }
2323
2324 /* Commit it to the VMCS and update our cache. */
2325 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2326 AssertRCReturn(rc, rc);
2327 pVCpu->hm.s.vmx.u32PinCtls = fVal;
2328
2329 return VINF_SUCCESS;
2330}
2331
2332
2333/**
2334 * Sets up secondary processor-based VM-execution controls in the VMCS.
2335 *
2336 * @returns VBox status code.
2337 * @param pVCpu The cross context virtual CPU structure.
2338 *
2339 * @remarks We don't really care about optimizing vmwrites here as it's done only
2340 * once per VM and hence we don't care about VMCS-field cache comparisons.
2341 */
2342static int hmR0VmxSetupProcCtls2(PVMCPU pVCpu)
2343{
2344 PVM pVM = pVCpu->CTX_SUFF(pVM);
2345 uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2346 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2347
2348 /* WBINVD causes a VM-exit. */
2349 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2350 fVal |= VMX_PROC_CTLS2_WBINVD_EXIT;
2351
2352 /* Enable EPT (aka nested-paging). */
2353 if (pVM->hm.s.fNestedPaging)
2354 fVal |= VMX_PROC_CTLS2_EPT;
2355
2356 /*
2357 * Enable the INVPCID instruction if supported by the hardware and we expose
2358 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2359 */
2360 if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
2361 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2362 fVal |= VMX_PROC_CTLS2_INVPCID;
2363
2364 /* Enable VPID. */
2365 if (pVM->hm.s.vmx.fVpid)
2366 fVal |= VMX_PROC_CTLS2_VPID;
2367
2368 /* Enable Unrestricted guest execution. */
2369 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2370 fVal |= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
2371
2372#if 0
2373 if (pVM->hm.s.fVirtApicRegs)
2374 {
2375 /* Enable APIC-register virtualization. */
2376 Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
2377 fVal |= VMX_PROC_CTLS2_APIC_REG_VIRT;
2378
2379 /* Enable virtual-interrupt delivery. */
2380 Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
2381 fVal |= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
2382 }
2383#endif
2384
2385 /* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
2386 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2387 * done dynamically. */
2388 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
2389 {
2390 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2391 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2392 fVal |= VMX_PROC_CTLS2_VIRT_APIC_ACCESS; /* Virtualize APIC accesses. */
2393 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2394 AssertRCReturn(rc, rc);
2395 }
2396
2397 /* Enable RDTSCP. */
2398 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
2399 fVal |= VMX_PROC_CTLS2_RDTSCP;
2400
2401 /* Enable Pause-Loop exiting. */
2402 if ( pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT
2403 && pVM->hm.s.vmx.cPleGapTicks
2404 && pVM->hm.s.vmx.cPleWindowTicks)
2405 {
2406 fVal |= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
2407
2408 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2409 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2410 AssertRCReturn(rc, rc);
2411 }
2412
2413 if ((fVal & fZap) != fVal)
2414 {
2415 LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2416 pVM->hm.s.vmx.Msrs.ProcCtls2.n.disallowed0, fVal, fZap));
2417 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2418 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2419 }
2420
2421 /* Commit it to the VMCS and update our cache. */
2422 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2423 AssertRCReturn(rc, rc);
2424 pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2425
2426 return VINF_SUCCESS;
2427}
2428
2429
2430/**
2431 * Sets up processor-based VM-execution controls in the VMCS.
2432 *
2433 * @returns VBox status code.
2434 * @param pVCpu The cross context virtual CPU structure.
2435 *
2436 * @remarks We don't really care about optimizing vmwrites here as it's done only
2437 * once per VM and hence we don't care about VMCS-field cache comparisons.
2438 */
2439static int hmR0VmxSetupProcCtls(PVMCPU pVCpu)
2440{
2441 PVM pVM = pVCpu->CTX_SUFF(pVM);
2442 uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2443 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2444
2445 fVal |= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
2446 | VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2447 | VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2448 | VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2449 | VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2450 | VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2451 | VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2452
2453 /* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2454 if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
2455 || (pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
2456 {
2457 LogRelFunc(("Unsupported VMX_PROC_CTLS_MOV_DR_EXIT combo!"));
2458 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2459 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2460 }
2461
2462 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2463 if (!pVM->hm.s.fNestedPaging)
2464 {
2465 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2466 fVal |= VMX_PROC_CTLS_INVLPG_EXIT
2467 | VMX_PROC_CTLS_CR3_LOAD_EXIT
2468 | VMX_PROC_CTLS_CR3_STORE_EXIT;
2469 }
2470
2471 /* Use TPR shadowing if supported by the CPU. */
2472 if ( PDMHasApic(pVM)
2473 && pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
2474 {
2475 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2476 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2477 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2478 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2479 AssertRCReturn(rc, rc);
2480
2481 fVal |= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2482 /* CR8 writes cause a VM-exit based on TPR threshold. */
2483 Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
2484 Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
2485 }
2486 else
2487 {
2488 /*
2489 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2490 * Set this control only for 64-bit guests.
2491 */
2492 if (pVM->hm.s.fAllow64BitGuests)
2493 {
2494 fVal |= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2495 | VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2496 }
2497 }
2498
2499 /* Use MSR-bitmaps if supported by the CPU. */
2500 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2501 {
2502 fVal |= VMX_PROC_CTLS_USE_MSR_BITMAPS;
2503
2504 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2505 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2506 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2507 AssertRCReturn(rc, rc);
2508
2509 /*
2510 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2511 * automatically using dedicated fields in the VMCS.
2512 */
2513 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2514 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2515 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2516 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2517 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2518#if HC_ARCH_BITS == 64
2519 /*
2520 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2521 */
2522 if (pVM->hm.s.fAllow64BitGuests)
2523 {
2524 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2525 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2526 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2527 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2528 }
2529#endif
2530 /*
2531 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2532 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2533 */
2534 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2535 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2536
2537 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2538 }
2539
2540 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2541 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2542 fVal |= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
2543
2544 if ((fVal & fZap) != fVal)
2545 {
2546 LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2547 pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0, fVal, fZap));
2548 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2549 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2550 }
2551
2552 /* Commit it to the VMCS and update our cache. */
2553 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2554 AssertRCReturn(rc, rc);
2555 pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2556
2557 /* Set up secondary processor-based VM-execution controls if the CPU supports it. */
2558 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2559 return hmR0VmxSetupProcCtls2(pVCpu);
2560
2561 /* Sanity check, should not really happen. */
2562 if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2563 {
2564 LogRelFunc(("Unrestricted Guest enabled when secondary processor-based VM-execution controls not available\n"));
2565 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2566 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2567 }
2568
2569 /* Old CPUs without secondary processor-based VM-execution controls would end up here. */
2570 return VINF_SUCCESS;
2571}
2572
2573
2574/**
2575 * Sets up miscellaneous (everything other than Pin & Processor-based
2576 * VM-execution) control fields in the VMCS.
2577 *
2578 * @returns VBox status code.
2579 * @param pVCpu The cross context virtual CPU structure.
2580 */
2581static int hmR0VmxSetupMiscCtls(PVMCPU pVCpu)
2582{
2583 AssertPtr(pVCpu);
2584
2585 int rc = VERR_GENERAL_FAILURE;
2586
2587 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2588#if 0
2589 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2590 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2591 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2592
2593 /*
2594 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2595 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2596 * We thus use the exception bitmap to control it rather than use both.
2597 */
2598 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2599 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2600
2601 /* All IO & IOIO instructions cause VM-exits. */
2602 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2603 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2604
2605 /* Initialize the MSR-bitmap area. */
2606 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2607 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2608 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2609 AssertRCReturn(rc, rc);
2610#endif
2611
2612 /* Setup MSR auto-load/store area. */
2613 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2614 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2615 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2616 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2617 AssertRCReturn(rc, rc);
2618
2619 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2620 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2621 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2622 AssertRCReturn(rc, rc);
2623
2624 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2625 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2626 AssertRCReturn(rc, rc);
2627
2628 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2629#if 0
2630 /* Setup debug controls */
2631 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2632 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2633 AssertRCReturn(rc, rc);
2634#endif
2635
2636 return rc;
2637}
2638
2639
2640/**
2641 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2642 *
2643 * We shall setup those exception intercepts that don't change during the
2644 * lifetime of the VM here. The rest are done dynamically while loading the
2645 * guest state.
2646 *
2647 * @returns VBox status code.
2648 * @param pVCpu The cross context virtual CPU structure.
2649 */
2650static int hmR0VmxInitXcptBitmap(PVMCPU pVCpu)
2651{
2652 AssertPtr(pVCpu);
2653
2654 uint32_t uXcptBitmap;
2655
2656 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2657 uXcptBitmap = RT_BIT_32(X86_XCPT_AC);
2658
2659 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2660 and writes, and because recursive #DBs can cause the CPU hang, we must always
2661 intercept #DB. */
2662 uXcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2663
2664 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2665 if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
2666 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
2667
2668 /* Commit it to the VMCS. */
2669 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2670 AssertRCReturn(rc, rc);
2671
2672 /* Update our cache of the exception bitmap. */
2673 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
2674 return VINF_SUCCESS;
2675}
2676
2677
2678/**
2679 * Does per-VM VT-x initialization.
2680 *
2681 * @returns VBox status code.
2682 * @param pVM The cross context VM structure.
2683 */
2684VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2685{
2686 LogFlowFunc(("pVM=%p\n", pVM));
2687
2688 int rc = hmR0VmxStructsAlloc(pVM);
2689 if (RT_FAILURE(rc))
2690 {
2691 LogRelFunc(("hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2692 return rc;
2693 }
2694
2695 return VINF_SUCCESS;
2696}
2697
2698
2699/**
2700 * Does per-VM VT-x termination.
2701 *
2702 * @returns VBox status code.
2703 * @param pVM The cross context VM structure.
2704 */
2705VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2706{
2707 LogFlowFunc(("pVM=%p\n", pVM));
2708
2709#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2710 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2711 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2712#endif
2713 hmR0VmxStructsFree(pVM);
2714 return VINF_SUCCESS;
2715}
2716
2717
2718/**
2719 * Sets up the VM for execution under VT-x.
2720 * This function is only called once per-VM during initialization.
2721 *
2722 * @returns VBox status code.
2723 * @param pVM The cross context VM structure.
2724 */
2725VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2726{
2727 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2728 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2729
2730 LogFlowFunc(("pVM=%p\n", pVM));
2731
2732 /*
2733 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be
2734 * allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2735 * pRealModeTSS, see hmR3InitFinalizeR0Intel().
2736 */
2737 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2738 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2739 || !pVM->hm.s.vmx.pRealModeTSS))
2740 {
2741 LogRelFunc(("Invalid real-on-v86 state.\n"));
2742 return VERR_INTERNAL_ERROR;
2743 }
2744
2745 /* Initialize these always, see hmR3InitFinalizeR0().*/
2746 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
2747 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
2748
2749 /* Setup the tagged-TLB flush handlers. */
2750 int rc = hmR0VmxSetupTaggedTlb(pVM);
2751 if (RT_FAILURE(rc))
2752 {
2753 LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2754 return rc;
2755 }
2756
2757 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2758 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2759#if HC_ARCH_BITS == 64
2760 if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
2761 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2762 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
2763 {
2764 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2765 }
2766#endif
2767
2768 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2769 RTCCUINTREG const uHostCR4 = ASMGetCR4();
2770 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2771 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2772
2773 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2774 {
2775 PVMCPU pVCpu = &pVM->aCpus[i];
2776 AssertPtr(pVCpu);
2777 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2778
2779 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2780 Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2781
2782 /* Set revision dword at the beginning of the VMCS structure. */
2783 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
2784
2785 /* Set the VMCS launch state to "clear", see Intel spec. 31.6 "Preparation and launch a virtual machine". */
2786 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2787 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc\n", rc),
2788 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2789
2790 /* Load this VMCS as the current VMCS. */
2791 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2792 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc\n", rc),
2793 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2794
2795 rc = hmR0VmxSetupPinCtls(pVCpu);
2796 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc\n", rc),
2797 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2798
2799 rc = hmR0VmxSetupProcCtls(pVCpu);
2800 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc\n", rc),
2801 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2802
2803 rc = hmR0VmxSetupMiscCtls(pVCpu);
2804 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc\n", rc),
2805 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2806
2807 rc = hmR0VmxInitXcptBitmap(pVCpu);
2808 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc\n", rc),
2809 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2810
2811#if HC_ARCH_BITS == 32
2812 rc = hmR0VmxInitVmcsReadCache(pVCpu);
2813 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc\n", rc),
2814 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2815#endif
2816
2817 /* Sync any CPU internal VMCS data back into our VMCS in memory. */
2818 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2819 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc\n", rc),
2820 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2821
2822 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2823
2824 hmR0VmxUpdateErrorRecord(pVCpu, rc);
2825 }
2826
2827 return VINF_SUCCESS;
2828}
2829
2830
2831/**
2832 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2833 * the VMCS.
2834 *
2835 * @returns VBox status code.
2836 */
2837static int hmR0VmxExportHostControlRegs(void)
2838{
2839 RTCCUINTREG uReg = ASMGetCR0();
2840 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2841 AssertRCReturn(rc, rc);
2842
2843 uReg = ASMGetCR3();
2844 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2845 AssertRCReturn(rc, rc);
2846
2847 uReg = ASMGetCR4();
2848 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2849 AssertRCReturn(rc, rc);
2850 return rc;
2851}
2852
2853
2854/**
2855 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2856 * the host-state area in the VMCS.
2857 *
2858 * @returns VBox status code.
2859 * @param pVCpu The cross context virtual CPU structure.
2860 */
2861static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2862{
2863#if HC_ARCH_BITS == 64
2864/**
2865 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2866 * requirements. See hmR0VmxExportHostSegmentRegs().
2867 */
2868# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2869 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2870 { \
2871 bool fValidSelector = true; \
2872 if ((selValue) & X86_SEL_LDT) \
2873 { \
2874 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2875 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2876 } \
2877 if (fValidSelector) \
2878 { \
2879 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2880 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2881 } \
2882 (selValue) = 0; \
2883 }
2884
2885 /*
2886 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2887 * should -not- save the messed up state without restoring the original host-state,
2888 * see @bugref{7240}.
2889 *
2890 * This apparently can happen (most likely the FPU changes), deal with it rather than
2891 * asserting. Was observed booting Solaris 10u10 32-bit guest.
2892 */
2893 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2894 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2895 {
2896 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2897 pVCpu->idCpu));
2898 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2899 }
2900 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2901#else
2902 RT_NOREF(pVCpu);
2903#endif
2904
2905 /*
2906 * Host DS, ES, FS and GS segment registers.
2907 */
2908#if HC_ARCH_BITS == 64
2909 RTSEL uSelDS = ASMGetDS();
2910 RTSEL uSelES = ASMGetES();
2911 RTSEL uSelFS = ASMGetFS();
2912 RTSEL uSelGS = ASMGetGS();
2913#else
2914 RTSEL uSelDS = 0;
2915 RTSEL uSelES = 0;
2916 RTSEL uSelFS = 0;
2917 RTSEL uSelGS = 0;
2918#endif
2919
2920 /*
2921 * Host CS and SS segment registers.
2922 */
2923 RTSEL uSelCS = ASMGetCS();
2924 RTSEL uSelSS = ASMGetSS();
2925
2926 /*
2927 * Host TR segment register.
2928 */
2929 RTSEL uSelTR = ASMGetTR();
2930
2931#if HC_ARCH_BITS == 64
2932 /*
2933 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2934 * gain VM-entry and restore them before we get preempted.
2935 *
2936 * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2937 */
2938 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2939 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2940 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2941 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2942# undef VMXLOCAL_ADJUST_HOST_SEG
2943#endif
2944
2945 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2946 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2947 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2948 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2949 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2950 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2951 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2952 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2953 Assert(uSelCS);
2954 Assert(uSelTR);
2955
2956 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2957#if 0
2958 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE))
2959 Assert(uSelSS != 0);
2960#endif
2961
2962 /* Write these host selector fields into the host-state area in the VMCS. */
2963 int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2964 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2965#if HC_ARCH_BITS == 64
2966 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2967 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2968 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2969 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2970#else
2971 NOREF(uSelDS);
2972 NOREF(uSelES);
2973 NOREF(uSelFS);
2974 NOREF(uSelGS);
2975#endif
2976 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2977 AssertRCReturn(rc, rc);
2978
2979 /*
2980 * Host GDTR and IDTR.
2981 */
2982 RTGDTR Gdtr;
2983 RTIDTR Idtr;
2984 RT_ZERO(Gdtr);
2985 RT_ZERO(Idtr);
2986 ASMGetGDTR(&Gdtr);
2987 ASMGetIDTR(&Idtr);
2988 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2989 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2990 AssertRCReturn(rc, rc);
2991
2992#if HC_ARCH_BITS == 64
2993 /*
2994 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
2995 * them to the maximum limit (0xffff) on every VM-exit.
2996 */
2997 if (Gdtr.cbGdt != 0xffff)
2998 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2999
3000 /*
3001 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
3002 * Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
3003 * as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
3004 * However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
3005 * uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
3006 * alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
3007 * at 0xffff on hosts where we are sure it won't cause trouble.
3008 */
3009# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3010 if (Idtr.cbIdt < 0x0fff)
3011# else
3012 if (Idtr.cbIdt != 0xffff)
3013# endif
3014 {
3015 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3016 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3017 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3018 }
3019#endif
3020
3021 /*
3022 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3023 * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3024 * RPL should be too in most cases.
3025 */
3026 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3027 ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3028
3029 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3030#if HC_ARCH_BITS == 64
3031 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3032
3033 /*
3034 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3035 * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3036 * restoration if the host has something else. Task switching is not supported in 64-bit
3037 * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3038 * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3039 *
3040 * [1] See Intel spec. 3.5 "System Descriptor Types".
3041 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3042 */
3043 PVM pVM = pVCpu->CTX_SUFF(pVM);
3044 Assert(pDesc->System.u4Type == 11);
3045 if ( pDesc->System.u16LimitLow != 0x67
3046 || pDesc->System.u4LimitHigh)
3047 {
3048 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3049 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3050 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3051 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3052 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3053 }
3054
3055 /*
3056 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3057 */
3058 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3059 {
3060 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3061 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3062 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3063 {
3064 /* The GDT is read-only but the writable GDT is available. */
3065 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3066 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3067 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3068 AssertRCReturn(rc, rc);
3069 }
3070 }
3071#else
3072 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3073#endif
3074 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3075 AssertRCReturn(rc, rc);
3076
3077 /*
3078 * Host FS base and GS base.
3079 */
3080#if HC_ARCH_BITS == 64
3081 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3082 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3083 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3084 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3085 AssertRCReturn(rc, rc);
3086
3087 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3088 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3089 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3090 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3091 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3092#endif
3093 return VINF_SUCCESS;
3094}
3095
3096
3097/**
3098 * Exports certain host MSRs in the VM-exit MSR-load area and some in the
3099 * host-state area of the VMCS.
3100 *
3101 * Theses MSRs will be automatically restored on the host after every successful
3102 * VM-exit.
3103 *
3104 * @returns VBox status code.
3105 * @param pVCpu The cross context virtual CPU structure.
3106 *
3107 * @remarks No-long-jump zone!!!
3108 */
3109static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3110{
3111 AssertPtr(pVCpu);
3112 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3113
3114 /*
3115 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3116 * rather than swapping them on every VM-entry.
3117 */
3118 hmR0VmxLazySaveHostMsrs(pVCpu);
3119
3120 /*
3121 * Host Sysenter MSRs.
3122 */
3123 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3124#if HC_ARCH_BITS == 32
3125 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3126 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3127#else
3128 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3129 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3130#endif
3131 AssertRCReturn(rc, rc);
3132
3133 /*
3134 * Host EFER MSR.
3135 *
3136 * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3137 * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3138 */
3139 PVM pVM = pVCpu->CTX_SUFF(pVM);
3140 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3141 {
3142 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3143 AssertRCReturn(rc, rc);
3144 }
3145
3146 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3147
3148 return VINF_SUCCESS;
3149}
3150
3151
3152/**
3153 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3154 *
3155 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3156 * these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3157 * hmR0VMxExportGuestEntryCtls().
3158 *
3159 * @returns true if we need to load guest EFER, false otherwise.
3160 * @param pVCpu The cross context virtual CPU structure.
3161 *
3162 * @remarks Requires EFER, CR4.
3163 * @remarks No-long-jump zone!!!
3164 */
3165static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu)
3166{
3167#ifdef HMVMX_ALWAYS_SWAP_EFER
3168 RT_NOREF(pVCpu);
3169 return true;
3170#else
3171
3172 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3173#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3174 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3175 if (CPUMIsGuestInLongModeEx(pCtx))
3176 return false;
3177#endif
3178
3179 PVM pVM = pVCpu->CTX_SUFF(pVM);
3180 uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3181 uint64_t const u64GuestEfer = pCtx->msrEFER;
3182
3183 /*
3184 * For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3185 * guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3186 */
3187 if ( CPUMIsGuestInLongModeEx(pCtx)
3188 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3189 {
3190 return true;
3191 }
3192
3193 /*
3194 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3195 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3196 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3197 */
3198 if ( (pCtx->cr4 & X86_CR4_PAE)
3199 && (pCtx->cr0 & X86_CR0_PG)
3200 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3201 {
3202 /* Assert that host is NX capable. */
3203 Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
3204 return true;
3205 }
3206
3207 return false;
3208#endif
3209}
3210
3211
3212/**
3213 * Exports the guest state with appropriate VM-entry controls in the VMCS.
3214 *
3215 * These controls can affect things done on VM-exit; e.g. "load debug controls",
3216 * see Intel spec. 24.8.1 "VM-entry controls".
3217 *
3218 * @returns VBox status code.
3219 * @param pVCpu The cross context virtual CPU structure.
3220 *
3221 * @remarks Requires EFER.
3222 * @remarks No-long-jump zone!!!
3223 */
3224static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu)
3225{
3226 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3227 {
3228 PVM pVM = pVCpu->CTX_SUFF(pVM);
3229 uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
3230 uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3231
3232 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3233 fVal |= VMX_ENTRY_CTLS_LOAD_DEBUG;
3234
3235 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3236 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
3237 {
3238 fVal |= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
3239 Log4Func(("VMX_ENTRY_CTLS_IA32E_MODE_GUEST\n"));
3240 }
3241 else
3242 Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
3243
3244 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3245 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3246 && hmR0VmxShouldSwapEferMsr(pVCpu))
3247 {
3248 fVal |= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
3249 Log4Func(("VMX_ENTRY_CTLS_LOAD_EFER_MSR\n"));
3250 }
3251
3252 /*
3253 * The following should -not- be set (since we're not in SMM mode):
3254 * - VMX_ENTRY_CTLS_ENTRY_TO_SMM
3255 * - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
3256 */
3257
3258 /** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
3259 * VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
3260
3261 if ((fVal & fZap) != fVal)
3262 {
3263 Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3264 pVM->hm.s.vmx.Msrs.EntryCtls.n.disallowed0, fVal, fZap));
3265 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3266 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3267 }
3268
3269 /* Commit it to the VMCS and update our cache. */
3270 if (pVCpu->hm.s.vmx.u32EntryCtls != fVal)
3271 {
3272 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3273 AssertRCReturn(rc, rc);
3274 pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3275 }
3276
3277 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3278 }
3279 return VINF_SUCCESS;
3280}
3281
3282
3283/**
3284 * Exports the guest state with appropriate VM-exit controls in the VMCS.
3285 *
3286 * @returns VBox status code.
3287 * @param pVCpu The cross context virtual CPU structure.
3288 *
3289 * @remarks Requires EFER.
3290 */
3291static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu)
3292{
3293 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3294 {
3295 PVM pVM = pVCpu->CTX_SUFF(pVM);
3296 uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
3297 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3298
3299 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3300 fVal |= VMX_EXIT_CTLS_SAVE_DEBUG;
3301
3302 /*
3303 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3304 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3305 * hmR0VmxExportHostMsrs().
3306 */
3307#if HC_ARCH_BITS == 64
3308 fVal |= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
3309 Log4Func(("VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE\n"));
3310#else
3311 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3312 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3313 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3314 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3315 {
3316 /* The switcher returns to long mode, EFER is managed by the switcher. */
3317 fVal |= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
3318 Log4Func(("VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE\n"));
3319 }
3320 else
3321 Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
3322#endif
3323
3324 /* If the newer VMCS fields for managing EFER exists, use it. */
3325 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3326 && hmR0VmxShouldSwapEferMsr(pVCpu))
3327 {
3328 fVal |= VMX_EXIT_CTLS_SAVE_EFER_MSR
3329 | VMX_EXIT_CTLS_LOAD_EFER_MSR;
3330 Log4Func(("VMX_EXIT_CTLS_SAVE_EFER_MSR and VMX_EXIT_CTLS_LOAD_EFER_MSR\n"));
3331 }
3332
3333 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3334 Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
3335
3336 /** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
3337 * VMX_EXIT_CTLS_SAVE_PAT_MSR,
3338 * VMX_EXIT_CTLS_LOAD_PAT_MSR. */
3339
3340 /* Enable saving of the VMX preemption timer value on VM-exit. */
3341 if ( pVM->hm.s.vmx.fUsePreemptTimer
3342 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
3343 fVal |= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
3344
3345 if ((fVal & fZap) != fVal)
3346 {
3347 LogRelFunc(("Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3348 pVM->hm.s.vmx.Msrs.ExitCtls.n.disallowed0, fVal, fZap));
3349 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3350 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3351 }
3352
3353 /* Commit it to the VMCS and update our cache. */
3354 if (pVCpu->hm.s.vmx.u32ExitCtls != fVal)
3355 {
3356 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3357 AssertRCReturn(rc, rc);
3358 pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3359 }
3360
3361 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3362 }
3363 return VINF_SUCCESS;
3364}
3365
3366
3367/**
3368 * Sets the TPR threshold in the VMCS.
3369 *
3370 * @returns VBox status code.
3371 * @param pVCpu The cross context virtual CPU structure.
3372 * @param u32TprThreshold The TPR threshold (task-priority class only).
3373 */
3374DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3375{
3376 Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
3377 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3378 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3379}
3380
3381
3382/**
3383 * Exports the guest APIC TPR state into the VMCS.
3384 *
3385 * @returns VBox status code.
3386 * @param pVCpu The cross context virtual CPU structure.
3387 *
3388 * @remarks No-long-jump zone!!!
3389 */
3390static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3391{
3392 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3393 {
3394 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
3395
3396 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3397 && APICIsEnabled(pVCpu))
3398 {
3399 /*
3400 * Setup TPR shadowing.
3401 */
3402 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
3403 {
3404 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3405
3406 bool fPendingIntr = false;
3407 uint8_t u8Tpr = 0;
3408 uint8_t u8PendingIntr = 0;
3409 int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3410 AssertRCReturn(rc, rc);
3411
3412 /*
3413 * If there are interrupts pending but masked by the TPR, instruct VT-x to
3414 * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3415 * priority of the pending interrupt so we can deliver the interrupt. If there
3416 * are no interrupts pending, set threshold to 0 to not cause any
3417 * TPR-below-threshold VM-exits.
3418 */
3419 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3420 uint32_t u32TprThreshold = 0;
3421 if (fPendingIntr)
3422 {
3423 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3424 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3425 const uint8_t u8TprPriority = u8Tpr >> 4;
3426 if (u8PendingPriority <= u8TprPriority)
3427 u32TprThreshold = u8PendingPriority;
3428 }
3429
3430 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3431 AssertRCReturn(rc, rc);
3432 }
3433 }
3434 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3435 }
3436 return VINF_SUCCESS;
3437}
3438
3439
3440/**
3441 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3442 *
3443 * @returns Guest's interruptibility-state.
3444 * @param pVCpu The cross context virtual CPU structure.
3445 *
3446 * @remarks No-long-jump zone!!!
3447 */
3448static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu)
3449{
3450 /*
3451 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3452 */
3453 uint32_t fIntrState = 0;
3454 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3455 {
3456 /* If inhibition is active, RIP & RFLAGS should've been accessed
3457 (i.e. read previously from the VMCS or from ring-3). */
3458 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3459#ifdef VBOX_STRICT
3460 uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
3461 AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3462#endif
3463 if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3464 {
3465 if (pCtx->eflags.Bits.u1IF)
3466 fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
3467 else
3468 fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
3469 }
3470 else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3471 {
3472 /*
3473 * We can clear the inhibit force flag as even if we go back to the recompiler
3474 * without executing guest code in VT-x, the flag's condition to be cleared is
3475 * met and thus the cleared state is correct.
3476 */
3477 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3478 }
3479 }
3480
3481 /*
3482 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3483 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3484 * setting this would block host-NMIs and IRET will not clear the blocking.
3485 *
3486 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3487 */
3488 if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS)
3489 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
3490 {
3491 fIntrState |= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
3492 }
3493
3494 return fIntrState;
3495}
3496
3497
3498/**
3499 * Exports the guest's interruptibility-state into the guest-state area in the
3500 * VMCS.
3501 *
3502 * @returns VBox status code.
3503 * @param pVCpu The cross context virtual CPU structure.
3504 * @param fIntrState The interruptibility-state to set.
3505 */
3506static int hmR0VmxExportGuestIntrState(PVMCPU pVCpu, uint32_t fIntrState)
3507{
3508 NOREF(pVCpu);
3509 AssertMsg(!(fIntrState & 0xfffffff0), ("%#x\n", fIntrState)); /* Bits 31:4 MBZ. */
3510 Assert((fIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3511 return VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
3512}
3513
3514
3515/**
3516 * Exports the exception intercepts required for guest execution in the VMCS.
3517 *
3518 * @returns VBox status code.
3519 * @param pVCpu The cross context virtual CPU structure.
3520 *
3521 * @remarks No-long-jump zone!!!
3522 */
3523static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3524{
3525 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3526 {
3527 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3528
3529 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3530 if (pVCpu->hm.s.fGIMTrapXcptUD)
3531 uXcptBitmap |= RT_BIT(X86_XCPT_UD);
3532#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3533 else
3534 uXcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3535#endif
3536
3537 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_AC));
3538 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_DB));
3539
3540 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3541 {
3542 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3543 AssertRCReturn(rc, rc);
3544 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3545 }
3546
3547 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3548 Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", uXcptBitmap));
3549 }
3550 return VINF_SUCCESS;
3551}
3552
3553
3554/**
3555 * Exports the guest's RIP into the guest-state area in the VMCS.
3556 *
3557 * @returns VBox status code.
3558 * @param pVCpu The cross context virtual CPU structure.
3559 *
3560 * @remarks No-long-jump zone!!!
3561 */
3562static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
3563{
3564 int rc = VINF_SUCCESS;
3565 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3566 {
3567 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
3568
3569 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
3570 AssertRCReturn(rc, rc);
3571
3572 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3573 Log4Func(("RIP=%#RX64\n", pVCpu->cpum.GstCtx.rip));
3574 }
3575 return rc;
3576}
3577
3578
3579/**
3580 * Exports the guest's RSP into the guest-state area in the VMCS.
3581 *
3582 * @returns VBox status code.
3583 * @param pVCpu The cross context virtual CPU structure.
3584 *
3585 * @remarks No-long-jump zone!!!
3586 */
3587static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
3588{
3589 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3590 {
3591 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
3592
3593 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
3594 AssertRCReturn(rc, rc);
3595
3596 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3597 }
3598 return VINF_SUCCESS;
3599}
3600
3601
3602/**
3603 * Exports the guest's RFLAGS into the guest-state area in the VMCS.
3604 *
3605 * @returns VBox status code.
3606 * @param pVCpu The cross context virtual CPU structure.
3607 *
3608 * @remarks No-long-jump zone!!!
3609 */
3610static int hmR0VmxExportGuestRflags(PVMCPU pVCpu)
3611{
3612 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3613 {
3614 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
3615
3616 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3617 Let us assert it as such and use 32-bit VMWRITE. */
3618 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
3619 X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
3620 Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3621 Assert(!(fEFlags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3622
3623 /*
3624 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3625 * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3626 * can run the real-mode guest code under Virtual 8086 mode.
3627 */
3628 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3629 {
3630 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3631 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3632 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3633 fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3634 fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3635 }
3636
3637 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3638 AssertRCReturn(rc, rc);
3639
3640 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3641 Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3642 }
3643 return VINF_SUCCESS;
3644}
3645
3646
3647/**
3648 * Exports the guest CR0 control register into the guest-state area in the VMCS.
3649 *
3650 * The guest FPU state is always pre-loaded hence we don't need to bother about
3651 * sharing FPU related CR0 bits between the guest and host.
3652 *
3653 * @returns VBox status code.
3654 * @param pVCpu The cross context virtual CPU structure.
3655 *
3656 * @remarks No-long-jump zone!!!
3657 */
3658static int hmR0VmxExportGuestCR0(PVMCPU pVCpu)
3659{
3660 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3661 {
3662 PVM pVM = pVCpu->CTX_SUFF(pVM);
3663 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
3664 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.cr0));
3665
3666 uint32_t const u32ShadowCr0 = pVCpu->cpum.GstCtx.cr0;
3667 uint32_t u32GuestCr0 = pVCpu->cpum.GstCtx.cr0;
3668
3669 /*
3670 * Setup VT-x's view of the guest CR0.
3671 * Minimize VM-exits due to CR3 changes when we have NestedPaging.
3672 */
3673 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3674 if (pVM->hm.s.fNestedPaging)
3675 {
3676 if (CPUMIsGuestPagingEnabled(pVCpu))
3677 {
3678 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3679 uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
3680 | VMX_PROC_CTLS_CR3_STORE_EXIT);
3681 }
3682 else
3683 {
3684 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3685 uProcCtls |= VMX_PROC_CTLS_CR3_LOAD_EXIT
3686 | VMX_PROC_CTLS_CR3_STORE_EXIT;
3687 }
3688
3689 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3690 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3691 uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
3692 }
3693 else
3694 {
3695 /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3696 u32GuestCr0 |= X86_CR0_WP;
3697 }
3698
3699 /*
3700 * Guest FPU bits.
3701 *
3702 * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3703 * using CR0.TS.
3704 *
3705 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3706 * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3707 */
3708 u32GuestCr0 |= X86_CR0_NE;
3709
3710 /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3711 bool const fInterceptMF = !(u32ShadowCr0 & X86_CR0_NE);
3712
3713 /*
3714 * Update exception intercepts.
3715 */
3716 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3717 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3718 {
3719 Assert(PDMVmmDevHeapIsEnabled(pVM));
3720 Assert(pVM->hm.s.vmx.pRealModeTSS);
3721 uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3722 }
3723 else
3724 {
3725 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3726 uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3727 if (fInterceptMF)
3728 uXcptBitmap |= RT_BIT(X86_XCPT_MF);
3729 }
3730
3731 /* Additional intercepts for debugging, define these yourself explicitly. */
3732#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3733 uXcptBitmap |= 0
3734 | RT_BIT(X86_XCPT_BP)
3735 | RT_BIT(X86_XCPT_DE)
3736 | RT_BIT(X86_XCPT_NM)
3737 | RT_BIT(X86_XCPT_TS)
3738 | RT_BIT(X86_XCPT_UD)
3739 | RT_BIT(X86_XCPT_NP)
3740 | RT_BIT(X86_XCPT_SS)
3741 | RT_BIT(X86_XCPT_GP)
3742 | RT_BIT(X86_XCPT_PF)
3743 | RT_BIT(X86_XCPT_MF)
3744 ;
3745#elif defined(HMVMX_ALWAYS_TRAP_PF)
3746 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
3747#endif
3748 if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
3749 uXcptBitmap |= RT_BIT(X86_XCPT_GP);
3750 Assert(pVM->hm.s.fNestedPaging || (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
3751
3752 /*
3753 * Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3754 */
3755 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3756 uint32_t fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3757 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3758 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
3759 else
3760 Assert((fSetCr0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3761
3762 u32GuestCr0 |= fSetCr0;
3763 u32GuestCr0 &= fZapCr0;
3764 u32GuestCr0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3765
3766 /*
3767 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3768 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3769 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3770 */
3771 uint32_t u32Cr0Mask = X86_CR0_PE
3772 | X86_CR0_NE
3773 | (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3774 | X86_CR0_PG
3775 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3776 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3777 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3778
3779 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3780 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3781 * and @bugref{6944}. */
3782#if 0
3783 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3784 u32Cr0Mask &= ~X86_CR0_PE;
3785#endif
3786 /*
3787 * Finally, update VMCS fields with the CR0 values and the exception bitmap.
3788 */
3789 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCr0);
3790 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32ShadowCr0);
3791 if (u32Cr0Mask != pVCpu->hm.s.vmx.u32Cr0Mask)
3792 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32Cr0Mask);
3793 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3794 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3795 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3796 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3797 AssertRCReturn(rc, rc);
3798
3799 /* Update our caches. */
3800 pVCpu->hm.s.vmx.u32Cr0Mask = u32Cr0Mask;
3801 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3802 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3803
3804 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3805
3806 Log4Func(("u32Cr0Mask=%#RX32 u32ShadowCr0=%#RX32 u32GuestCr0=%#RX32 (fSetCr0=%#RX32 fZapCr0=%#RX32\n", u32Cr0Mask,
3807 u32ShadowCr0, u32GuestCr0, fSetCr0, fZapCr0));
3808 }
3809
3810 return VINF_SUCCESS;
3811}
3812
3813
3814/**
3815 * Exports the guest control registers (CR3, CR4) into the guest-state area
3816 * in the VMCS.
3817 *
3818 * @returns VBox strict status code.
3819 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3820 * without unrestricted guest access and the VMMDev is not presently
3821 * mapped (e.g. EFI32).
3822 *
3823 * @param pVCpu The cross context virtual CPU structure.
3824 *
3825 * @remarks No-long-jump zone!!!
3826 */
3827static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu)
3828{
3829 int rc = VINF_SUCCESS;
3830 PVM pVM = pVCpu->CTX_SUFF(pVM);
3831
3832 /*
3833 * Guest CR2.
3834 * It's always loaded in the assembler code. Nothing to do here.
3835 */
3836
3837 /*
3838 * Guest CR3.
3839 */
3840 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3841 {
3842 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
3843
3844 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3845 if (pVM->hm.s.fNestedPaging)
3846 {
3847 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3848
3849 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3850 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3851 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3852 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3853
3854 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3855 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3856 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3857
3858 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3859 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3860 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3861 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3862 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3863 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3864 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3865
3866 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3867 AssertRCReturn(rc, rc);
3868
3869 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3870 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3871 || CPUMIsGuestPagingEnabledEx(pCtx))
3872 {
3873 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3874 if (CPUMIsGuestInPAEModeEx(pCtx))
3875 {
3876 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3877 AssertRCReturn(rc, rc);
3878 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3879 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3880 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3881 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3882 AssertRCReturn(rc, rc);
3883 }
3884
3885 /*
3886 * The guest's view of its CR3 is unblemished with Nested Paging when the
3887 * guest is using paging or we have unrestricted guest execution to handle
3888 * the guest when it's not using paging.
3889 */
3890 GCPhysGuestCR3 = pCtx->cr3;
3891 }
3892 else
3893 {
3894 /*
3895 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
3896 * thinks it accesses physical memory directly, we use our identity-mapped
3897 * page table to map guest-linear to guest-physical addresses. EPT takes care
3898 * of translating it to host-physical addresses.
3899 */
3900 RTGCPHYS GCPhys;
3901 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3902
3903 /* We obtain it here every time as the guest could have relocated this PCI region. */
3904 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3905 if (RT_SUCCESS(rc))
3906 { /* likely */ }
3907 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3908 {
3909 Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3910 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3911 }
3912 else
3913 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3914
3915 GCPhysGuestCR3 = GCPhys;
3916 }
3917
3918 Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
3919 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3920 AssertRCReturn(rc, rc);
3921 }
3922 else
3923 {
3924 /* Non-nested paging case, just use the hypervisor's CR3. */
3925 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3926
3927 Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
3928 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3929 AssertRCReturn(rc, rc);
3930 }
3931
3932 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3933 }
3934
3935 /*
3936 * Guest CR4.
3937 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3938 */
3939 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3940 {
3941 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3942 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
3943 Assert(!RT_HI_U32(pCtx->cr4));
3944
3945 uint32_t u32GuestCr4 = pCtx->cr4;
3946 uint32_t const u32ShadowCr4 = pCtx->cr4;
3947
3948 /*
3949 * Setup VT-x's view of the guest CR4.
3950 *
3951 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3952 * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3953 * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3954 *
3955 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3956 */
3957 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3958 {
3959 Assert(pVM->hm.s.vmx.pRealModeTSS);
3960 Assert(PDMVmmDevHeapIsEnabled(pVM));
3961 u32GuestCr4 &= ~X86_CR4_VME;
3962 }
3963
3964 if (pVM->hm.s.fNestedPaging)
3965 {
3966 if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3967 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3968 {
3969 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3970 u32GuestCr4 |= X86_CR4_PSE;
3971 /* Our identity mapping is a 32-bit page directory. */
3972 u32GuestCr4 &= ~X86_CR4_PAE;
3973 }
3974 /* else use guest CR4.*/
3975 }
3976 else
3977 {
3978 /*
3979 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3980 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3981 */
3982 switch (pVCpu->hm.s.enmShadowMode)
3983 {
3984 case PGMMODE_REAL: /* Real-mode. */
3985 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3986 case PGMMODE_32_BIT: /* 32-bit paging. */
3987 {
3988 u32GuestCr4 &= ~X86_CR4_PAE;
3989 break;
3990 }
3991
3992 case PGMMODE_PAE: /* PAE paging. */
3993 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3994 {
3995 u32GuestCr4 |= X86_CR4_PAE;
3996 break;
3997 }
3998
3999 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4000 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4001#ifdef VBOX_ENABLE_64_BITS_GUESTS
4002 break;
4003#endif
4004 default:
4005 AssertFailed();
4006 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4007 }
4008 }
4009
4010 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4011 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4012 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4013 u32GuestCr4 |= fSetCr4;
4014 u32GuestCr4 &= fZapCr4;
4015
4016 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them,
4017 that would cause a VM-exit. */
4018 uint32_t u32Cr4Mask = X86_CR4_VME
4019 | X86_CR4_PAE
4020 | X86_CR4_PGE
4021 | X86_CR4_PSE
4022 | X86_CR4_VMXE;
4023 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4024 u32Cr4Mask |= X86_CR4_OSXSAVE;
4025 if (pVM->cpum.ro.GuestFeatures.fPcid)
4026 u32Cr4Mask |= X86_CR4_PCIDE;
4027
4028 /* Write VT-x's view of the guest CR4, the CR4 modify mask and the read-only CR4 shadow
4029 into the VMCS and update our cache. */
4030 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCr4);
4031 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32ShadowCr4);
4032 if (pVCpu->hm.s.vmx.u32Cr4Mask != u32Cr4Mask)
4033 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32Cr4Mask);
4034 AssertRCReturn(rc, rc);
4035 pVCpu->hm.s.vmx.u32Cr4Mask = u32Cr4Mask;
4036
4037 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4038 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
4039
4040 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4041
4042 Log4Func(("u32GuestCr4=%#RX32 u32ShadowCr4=%#RX32 (fSetCr4=%#RX32 fZapCr4=%#RX32)\n", u32GuestCr4, u32ShadowCr4, fSetCr4,
4043 fZapCr4));
4044 }
4045 return rc;
4046}
4047
4048
4049/**
4050 * Exports the guest debug registers into the guest-state area in the VMCS.
4051 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4052 *
4053 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4054 *
4055 * @returns VBox status code.
4056 * @param pVCpu The cross context virtual CPU structure.
4057 *
4058 * @remarks No-long-jump zone!!!
4059 */
4060static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu)
4061{
4062 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4063
4064#ifdef VBOX_STRICT
4065 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4066 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
4067 {
4068 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4069 Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0);
4070 Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
4071 }
4072#endif
4073
4074 bool fSteppingDB = false;
4075 bool fInterceptMovDRx = false;
4076 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4077 if (pVCpu->hm.s.fSingleInstruction)
4078 {
4079 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4080 PVM pVM = pVCpu->CTX_SUFF(pVM);
4081 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
4082 {
4083 uProcCtls |= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
4084 Assert(fSteppingDB == false);
4085 }
4086 else
4087 {
4088 pVCpu->cpum.GstCtx.eflags.u32 |= X86_EFL_TF;
4089 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
4090 pVCpu->hm.s.fClearTrapFlag = true;
4091 fSteppingDB = true;
4092 }
4093 }
4094
4095 uint32_t u32GuestDr7;
4096 if ( fSteppingDB
4097 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4098 {
4099 /*
4100 * Use the combined guest and host DRx values found in the hypervisor register set
4101 * because the debugger has breakpoints active or someone is single stepping on the
4102 * host side without a monitor trap flag.
4103 *
4104 * Note! DBGF expects a clean DR6 state before executing guest code.
4105 */
4106#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4107 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4108 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4109 {
4110 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4111 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4112 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4113 }
4114 else
4115#endif
4116 if (!CPUMIsHyperDebugStateActive(pVCpu))
4117 {
4118 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4119 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4120 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4121 }
4122
4123 /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4124 u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4125 pVCpu->hm.s.fUsingHyperDR7 = true;
4126 fInterceptMovDRx = true;
4127 }
4128 else
4129 {
4130 /*
4131 * If the guest has enabled debug registers, we need to load them prior to
4132 * executing guest code so they'll trigger at the right time.
4133 */
4134 if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD))
4135 {
4136#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4137 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4138 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4139 {
4140 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4141 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4142 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4143 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4144 }
4145 else
4146#endif
4147 if (!CPUMIsGuestDebugStateActive(pVCpu))
4148 {
4149 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4150 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4151 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4152 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4153 }
4154 Assert(!fInterceptMovDRx);
4155 }
4156 /*
4157 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4158 * must intercept #DB in order to maintain a correct DR6 guest value, and
4159 * because we need to intercept it to prevent nested #DBs from hanging the
4160 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4161 */
4162#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4163 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4164 && !CPUMIsGuestDebugStateActive(pVCpu))
4165#else
4166 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4167#endif
4168 {
4169 fInterceptMovDRx = true;
4170 }
4171
4172 /* Update DR7 with the actual guest value. */
4173 u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
4174 pVCpu->hm.s.fUsingHyperDR7 = false;
4175 }
4176
4177 if (fInterceptMovDRx)
4178 uProcCtls |= VMX_PROC_CTLS_MOV_DR_EXIT;
4179 else
4180 uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
4181
4182 /*
4183 * Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
4184 * monitor-trap flag and update our cache.
4185 */
4186 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4187 {
4188 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4189 AssertRCReturn(rc2, rc2);
4190 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4191 }
4192
4193 /*
4194 * Update guest DR7.
4195 */
4196 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
4197 AssertRCReturn(rc, rc);
4198
4199 return VINF_SUCCESS;
4200}
4201
4202
4203#ifdef VBOX_STRICT
4204/**
4205 * Strict function to validate segment registers.
4206 *
4207 * @param pVCpu The cross context virtual CPU structure.
4208 *
4209 * @remarks Will import guest CR0 on strict builds during validation of
4210 * segments.
4211 */
4212static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu)
4213{
4214 /*
4215 * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4216 *
4217 * The reason we check for attribute value 0 in this function and not just the unusable bit is
4218 * because hmR0VmxExportGuestSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4219 * and doesn't change the guest-context value.
4220 */
4221 PVM pVM = pVCpu->CTX_SUFF(pVM);
4222 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4223 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4224 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4225 && ( !CPUMIsGuestInRealModeEx(pCtx)
4226 && !CPUMIsGuestInV86ModeEx(pCtx)))
4227 {
4228 /* Protected mode checks */
4229 /* CS */
4230 Assert(pCtx->cs.Attr.n.u1Present);
4231 Assert(!(pCtx->cs.Attr.u & 0xf00));
4232 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4233 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4234 || !(pCtx->cs.Attr.n.u1Granularity));
4235 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4236 || (pCtx->cs.Attr.n.u1Granularity));
4237 /* CS cannot be loaded with NULL in protected mode. */
4238 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4239 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4240 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4241 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4242 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4243 else
4244 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4245 /* SS */
4246 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4247 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4248 if ( !(pCtx->cr0 & X86_CR0_PE)
4249 || pCtx->cs.Attr.n.u4Type == 3)
4250 {
4251 Assert(!pCtx->ss.Attr.n.u2Dpl);
4252 }
4253 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4254 {
4255 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4256 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4257 Assert(pCtx->ss.Attr.n.u1Present);
4258 Assert(!(pCtx->ss.Attr.u & 0xf00));
4259 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4260 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4261 || !(pCtx->ss.Attr.n.u1Granularity));
4262 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4263 || (pCtx->ss.Attr.n.u1Granularity));
4264 }
4265 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmentReg(). */
4266 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4267 {
4268 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4269 Assert(pCtx->ds.Attr.n.u1Present);
4270 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4271 Assert(!(pCtx->ds.Attr.u & 0xf00));
4272 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4273 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4274 || !(pCtx->ds.Attr.n.u1Granularity));
4275 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4276 || (pCtx->ds.Attr.n.u1Granularity));
4277 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4278 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4279 }
4280 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4281 {
4282 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4283 Assert(pCtx->es.Attr.n.u1Present);
4284 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4285 Assert(!(pCtx->es.Attr.u & 0xf00));
4286 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4287 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4288 || !(pCtx->es.Attr.n.u1Granularity));
4289 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4290 || (pCtx->es.Attr.n.u1Granularity));
4291 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4292 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4293 }
4294 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4295 {
4296 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4297 Assert(pCtx->fs.Attr.n.u1Present);
4298 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4299 Assert(!(pCtx->fs.Attr.u & 0xf00));
4300 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4301 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4302 || !(pCtx->fs.Attr.n.u1Granularity));
4303 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4304 || (pCtx->fs.Attr.n.u1Granularity));
4305 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4306 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4307 }
4308 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4309 {
4310 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4311 Assert(pCtx->gs.Attr.n.u1Present);
4312 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4313 Assert(!(pCtx->gs.Attr.u & 0xf00));
4314 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4315 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4316 || !(pCtx->gs.Attr.n.u1Granularity));
4317 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4318 || (pCtx->gs.Attr.n.u1Granularity));
4319 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4320 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4321 }
4322 /* 64-bit capable CPUs. */
4323# if HC_ARCH_BITS == 64
4324 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4325 Assert(!pCtx->ss.Attr.u || !RT_HI_U32(pCtx->ss.u64Base));
4326 Assert(!pCtx->ds.Attr.u || !RT_HI_U32(pCtx->ds.u64Base));
4327 Assert(!pCtx->es.Attr.u || !RT_HI_U32(pCtx->es.u64Base));
4328# endif
4329 }
4330 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4331 || ( CPUMIsGuestInRealModeEx(pCtx)
4332 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4333 {
4334 /* Real and v86 mode checks. */
4335 /* hmR0VmxExportGuestSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4336 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4337 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4338 {
4339 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4340 }
4341 else
4342 {
4343 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4344 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4345 }
4346
4347 /* CS */
4348 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4349 Assert(pCtx->cs.u32Limit == 0xffff);
4350 Assert(u32CSAttr == 0xf3);
4351 /* SS */
4352 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4353 Assert(pCtx->ss.u32Limit == 0xffff);
4354 Assert(u32SSAttr == 0xf3);
4355 /* DS */
4356 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4357 Assert(pCtx->ds.u32Limit == 0xffff);
4358 Assert(u32DSAttr == 0xf3);
4359 /* ES */
4360 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4361 Assert(pCtx->es.u32Limit == 0xffff);
4362 Assert(u32ESAttr == 0xf3);
4363 /* FS */
4364 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4365 Assert(pCtx->fs.u32Limit == 0xffff);
4366 Assert(u32FSAttr == 0xf3);
4367 /* GS */
4368 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4369 Assert(pCtx->gs.u32Limit == 0xffff);
4370 Assert(u32GSAttr == 0xf3);
4371 /* 64-bit capable CPUs. */
4372# if HC_ARCH_BITS == 64
4373 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4374 Assert(!u32SSAttr || !RT_HI_U32(pCtx->ss.u64Base));
4375 Assert(!u32DSAttr || !RT_HI_U32(pCtx->ds.u64Base));
4376 Assert(!u32ESAttr || !RT_HI_U32(pCtx->es.u64Base));
4377# endif
4378 }
4379}
4380#endif /* VBOX_STRICT */
4381
4382
4383/**
4384 * Exports a guest segment register into the guest-state area in the VMCS.
4385 *
4386 * @returns VBox status code.
4387 * @param pVCpu The cross context virtual CPU structure.
4388 * @param idxSel Index of the selector in the VMCS.
4389 * @param idxLimit Index of the segment limit in the VMCS.
4390 * @param idxBase Index of the segment base in the VMCS.
4391 * @param idxAccess Index of the access rights of the segment in the VMCS.
4392 * @param pSelReg Pointer to the segment selector.
4393 *
4394 * @remarks No-long-jump zone!!!
4395 */
4396static int hmR0VmxExportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
4397 PCCPUMSELREG pSelReg)
4398{
4399 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4400 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4401 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4402 AssertRCReturn(rc, rc);
4403
4404 uint32_t u32Access = pSelReg->Attr.u;
4405 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4406 {
4407 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4408 u32Access = 0xf3;
4409 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4410 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4411 }
4412 else
4413 {
4414 /*
4415 * The way to differentiate between whether this is really a null selector or was just
4416 * a selector loaded with 0 in real-mode is using the segment attributes. A selector
4417 * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4418 * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4419 * NULL selectors loaded in protected-mode have their attribute as 0.
4420 */
4421 if (!u32Access)
4422 u32Access = X86DESCATTR_UNUSABLE;
4423 }
4424
4425 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4426 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4427 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4428
4429 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4430 AssertRCReturn(rc, rc);
4431 return rc;
4432}
4433
4434
4435/**
4436 * Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4437 * into the guest-state area in the VMCS.
4438 *
4439 * @returns VBox status code.
4440 * @param pVCpu The cross context virtual CPU structure.
4441 *
4442 * @remarks Will import guest CR0 on strict builds during validation of
4443 * segments.
4444 * @remarks No-long-jump zone!!!
4445 */
4446static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu)
4447{
4448 int rc = VERR_INTERNAL_ERROR_5;
4449 PVM pVM = pVCpu->CTX_SUFF(pVM);
4450 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4451
4452 /*
4453 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4454 */
4455 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4456 {
4457#ifdef VBOX_WITH_REM
4458 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4459 {
4460 Assert(pVM->hm.s.vmx.pRealModeTSS);
4461 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4462 if ( pVCpu->hm.s.vmx.fWasInRealMode
4463 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4464 {
4465 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4466 in real-mode (e.g. OpenBSD 4.0) */
4467 REMFlushTBs(pVM);
4468 Log4Func(("Switch to protected mode detected!\n"));
4469 pVCpu->hm.s.vmx.fWasInRealMode = false;
4470 }
4471 }
4472#endif
4473 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
4474 {
4475 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
4476 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4477 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pCtx->cs.Attr.u;
4478 rc = HMVMX_EXPORT_SREG(CS, &pCtx->cs);
4479 AssertRCReturn(rc, rc);
4480 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
4481 }
4482
4483 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
4484 {
4485 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
4486 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4487 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pCtx->ss.Attr.u;
4488 rc = HMVMX_EXPORT_SREG(SS, &pCtx->ss);
4489 AssertRCReturn(rc, rc);
4490 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
4491 }
4492
4493 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
4494 {
4495 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
4496 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4497 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pCtx->ds.Attr.u;
4498 rc = HMVMX_EXPORT_SREG(DS, &pCtx->ds);
4499 AssertRCReturn(rc, rc);
4500 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
4501 }
4502
4503 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
4504 {
4505 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
4506 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4507 pVCpu->hm.s.vmx.RealMode.AttrES.u = pCtx->es.Attr.u;
4508 rc = HMVMX_EXPORT_SREG(ES, &pCtx->es);
4509 AssertRCReturn(rc, rc);
4510 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
4511 }
4512
4513 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
4514 {
4515 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
4516 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4517 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pCtx->fs.Attr.u;
4518 rc = HMVMX_EXPORT_SREG(FS, &pCtx->fs);
4519 AssertRCReturn(rc, rc);
4520 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
4521 }
4522
4523 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
4524 {
4525 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
4526 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4527 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pCtx->gs.Attr.u;
4528 rc = HMVMX_EXPORT_SREG(GS, &pCtx->gs);
4529 AssertRCReturn(rc, rc);
4530 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
4531 }
4532
4533#ifdef VBOX_STRICT
4534 hmR0VmxValidateSegmentRegs(pVCpu);
4535#endif
4536
4537 Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pCtx->cs.Sel, pCtx->cs.u64Base,
4538 pCtx->cs.u32Limit, pCtx->cs.Attr.u));
4539 }
4540
4541 /*
4542 * Guest TR.
4543 */
4544 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4545 {
4546 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
4547
4548 /*
4549 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
4550 * achieved using the interrupt redirection bitmap (all bits cleared to let the guest
4551 * handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4552 */
4553 uint16_t u16Sel = 0;
4554 uint32_t u32Limit = 0;
4555 uint64_t u64Base = 0;
4556 uint32_t u32AccessRights = 0;
4557
4558 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4559 {
4560 u16Sel = pCtx->tr.Sel;
4561 u32Limit = pCtx->tr.u32Limit;
4562 u64Base = pCtx->tr.u64Base;
4563 u32AccessRights = pCtx->tr.Attr.u;
4564 }
4565 else
4566 {
4567 Assert(pVM->hm.s.vmx.pRealModeTSS);
4568 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
4569
4570 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4571 RTGCPHYS GCPhys;
4572 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4573 AssertRCReturn(rc, rc);
4574
4575 X86DESCATTR DescAttr;
4576 DescAttr.u = 0;
4577 DescAttr.n.u1Present = 1;
4578 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4579
4580 u16Sel = 0;
4581 u32Limit = HM_VTX_TSS_SIZE;
4582 u64Base = GCPhys; /* in real-mode phys = virt. */
4583 u32AccessRights = DescAttr.u;
4584 }
4585
4586 /* Validate. */
4587 Assert(!(u16Sel & RT_BIT(2)));
4588 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4589 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4590 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4591 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4592 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4593 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4594 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4595 Assert( (u32Limit & 0xfff) == 0xfff
4596 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4597 Assert( !(pCtx->tr.u32Limit & 0xfff00000)
4598 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4599
4600 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4601 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4602 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4603 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4604 AssertRCReturn(rc, rc);
4605
4606 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4607 Log4Func(("TR base=%#RX64\n", pCtx->tr.u64Base));
4608 }
4609
4610 /*
4611 * Guest GDTR.
4612 */
4613 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4614 {
4615 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
4616
4617 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
4618 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
4619 AssertRCReturn(rc, rc);
4620
4621 /* Validate. */
4622 Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4623
4624 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4625 Log4Func(("GDTR base=%#RX64\n", pCtx->gdtr.pGdt));
4626 }
4627
4628 /*
4629 * Guest LDTR.
4630 */
4631 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4632 {
4633 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
4634
4635 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4636 uint32_t u32Access = 0;
4637 if (!pCtx->ldtr.Attr.u)
4638 u32Access = X86DESCATTR_UNUSABLE;
4639 else
4640 u32Access = pCtx->ldtr.Attr.u;
4641
4642 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
4643 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
4644 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4645 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
4646 AssertRCReturn(rc, rc);
4647
4648 /* Validate. */
4649 if (!(u32Access & X86DESCATTR_UNUSABLE))
4650 {
4651 Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4652 Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4653 Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4654 Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4655 Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4656 Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4657 Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
4658 || !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4659 Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
4660 || pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4661 }
4662
4663 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4664 Log4Func(("LDTR base=%#RX64\n", pCtx->ldtr.u64Base));
4665 }
4666
4667 /*
4668 * Guest IDTR.
4669 */
4670 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4671 {
4672 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
4673
4674 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
4675 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
4676 AssertRCReturn(rc, rc);
4677
4678 /* Validate. */
4679 Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4680
4681 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4682 Log4Func(("IDTR base=%#RX64\n", pCtx->idtr.pIdt));
4683 }
4684
4685 return VINF_SUCCESS;
4686}
4687
4688
4689/**
4690 * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4691 * areas.
4692 *
4693 * These MSRs will automatically be loaded to the host CPU on every successful
4694 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4695 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4696 * -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4697 *
4698 * Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4699 *
4700 * @returns VBox status code.
4701 * @param pVCpu The cross context virtual CPU structure.
4702 *
4703 * @remarks No-long-jump zone!!!
4704 */
4705static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu)
4706{
4707 AssertPtr(pVCpu);
4708 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4709
4710 /*
4711 * MSRs that we use the auto-load/store MSR area in the VMCS.
4712 * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4713 */
4714 PVM pVM = pVCpu->CTX_SUFF(pVM);
4715 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4716 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4717 {
4718 if (pVM->hm.s.fAllow64BitGuests)
4719 {
4720#if HC_ARCH_BITS == 32
4721 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS | CPUMCTX_EXTRN_KERNEL_GS_BASE);
4722
4723 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pCtx->msrLSTAR, false, NULL);
4724 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pCtx->msrSTAR, false, NULL);
4725 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pCtx->msrSFMASK, false, NULL);
4726 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, false, NULL);
4727 AssertRCReturn(rc, rc);
4728# ifdef LOG_ENABLED
4729 PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4730 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4731 Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4732# endif
4733#endif
4734 }
4735 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4736 }
4737
4738 /*
4739 * Guest Sysenter MSRs.
4740 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4741 * VM-exits on WRMSRs for these MSRs.
4742 */
4743 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4744 {
4745 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
4746
4747 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4748 {
4749 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
4750 AssertRCReturn(rc, rc);
4751 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4752 }
4753
4754 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4755 {
4756 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
4757 AssertRCReturn(rc, rc);
4758 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4759 }
4760
4761 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4762 {
4763 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
4764 AssertRCReturn(rc, rc);
4765 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4766 }
4767 }
4768
4769 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4770 {
4771 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
4772
4773 if (hmR0VmxShouldSwapEferMsr(pVCpu))
4774 {
4775 /*
4776 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4777 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4778 */
4779 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4780 {
4781 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
4782 AssertRCReturn(rc,rc);
4783 Log4Func(("EFER=%#RX64\n", pCtx->msrEFER));
4784 }
4785 else
4786 {
4787 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pCtx->msrEFER, false /* fUpdateHostMsr */,
4788 NULL /* pfAddedAndUpdated */);
4789 AssertRCReturn(rc, rc);
4790
4791 /* We need to intercept reads too, see @bugref{7386#c16}. */
4792 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
4793 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4794 Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pCtx->msrEFER,
4795 pVCpu->hm.s.vmx.cMsrs));
4796 }
4797 }
4798 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4799 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4800 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4801 }
4802
4803 return VINF_SUCCESS;
4804}
4805
4806
4807#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4808/**
4809 * Check if guest state allows safe use of 32-bit switcher again.
4810 *
4811 * Segment bases and protected mode structures must be 32-bit addressable
4812 * because the 32-bit switcher will ignore high dword when writing these VMCS
4813 * fields. See @bugref{8432} for details.
4814 *
4815 * @returns true if safe, false if must continue to use the 64-bit switcher.
4816 * @param pCtx Pointer to the guest-CPU context.
4817 *
4818 * @remarks No-long-jump zone!!!
4819 */
4820static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
4821{
4822 if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4823 if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4824 if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4825 if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4826 if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4827 if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4828 if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4829 if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4830 if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4831 if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4832
4833 /* All good, bases are 32-bit. */
4834 return true;
4835}
4836#endif
4837
4838
4839/**
4840 * Selects up the appropriate function to run guest code.
4841 *
4842 * @returns VBox status code.
4843 * @param pVCpu The cross context virtual CPU structure.
4844 *
4845 * @remarks No-long-jump zone!!!
4846 */
4847static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu)
4848{
4849 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4850 if (CPUMIsGuestInLongModeEx(pCtx))
4851 {
4852#ifndef VBOX_ENABLE_64_BITS_GUESTS
4853 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4854#endif
4855 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4856#if HC_ARCH_BITS == 32
4857 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4858 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4859 {
4860#ifdef VBOX_STRICT
4861 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4862 {
4863 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4864 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4865 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4866 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4867 | HM_CHANGED_VMX_ENTRY_CTLS
4868 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4869 }
4870#endif
4871 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4872
4873 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4874 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4875 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4876 Log4Func(("Selected 64-bit switcher\n"));
4877 }
4878#else
4879 /* 64-bit host. */
4880 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4881#endif
4882 }
4883 else
4884 {
4885 /* Guest is not in long mode, use the 32-bit handler. */
4886#if HC_ARCH_BITS == 32
4887 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4888 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4889 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4890 {
4891# ifdef VBOX_STRICT
4892 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4893 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4894 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4895 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4896 | HM_CHANGED_VMX_ENTRY_CTLS
4897 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4898# endif
4899 }
4900# ifdef VBOX_ENABLE_64_BITS_GUESTS
4901 /*
4902 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4903 * design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4904 * switcher flag because now we know the guest is in a sane state where it's safe
4905 * to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4906 * the much faster 32-bit switcher again.
4907 */
4908 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4909 {
4910 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4911 Log4Func(("Selected 32-bit switcher\n"));
4912 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4913 }
4914 else
4915 {
4916 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4917 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4918 || hmR0VmxIs32BitSwitcherSafe(pCtx))
4919 {
4920 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4921 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4922 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4923 | HM_CHANGED_VMX_ENTRY_CTLS
4924 | HM_CHANGED_VMX_EXIT_CTLS
4925 | HM_CHANGED_HOST_CONTEXT);
4926 Log4Func(("Selected 32-bit switcher (safe)\n"));
4927 }
4928 }
4929# else
4930 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4931# endif
4932#else
4933 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4934#endif
4935 }
4936 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4937 return VINF_SUCCESS;
4938}
4939
4940
4941/**
4942 * Wrapper for running the guest code in VT-x.
4943 *
4944 * @returns VBox status code, no informational status codes.
4945 * @param pVCpu The cross context virtual CPU structure.
4946 *
4947 * @remarks No-long-jump zone!!!
4948 */
4949DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu)
4950{
4951 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4952 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4953 pCtx->fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
4954
4955 /*
4956 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4957 * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4958 * callee-saved and thus the need for this XMM wrapper.
4959 *
4960 * See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4961 */
4962 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4963 /** @todo Add stats for resume vs launch. */
4964 PVM pVM = pVCpu->CTX_SUFF(pVM);
4965#ifdef VBOX_WITH_KERNEL_USING_XMM
4966 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4967#else
4968 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4969#endif
4970 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4971 return rc;
4972}
4973
4974
4975/**
4976 * Reports world-switch error and dumps some useful debug info.
4977 *
4978 * @param pVCpu The cross context virtual CPU structure.
4979 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4980 * @param pVmxTransient Pointer to the VMX transient structure (only
4981 * exitReason updated).
4982 */
4983static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
4984{
4985 Assert(pVCpu);
4986 Assert(pVmxTransient);
4987 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
4988
4989 Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
4990 switch (rcVMRun)
4991 {
4992 case VERR_VMX_INVALID_VMXON_PTR:
4993 AssertFailed();
4994 break;
4995 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4996 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4997 {
4998 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4999 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5000 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
5001 AssertRC(rc);
5002
5003 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5004 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5005 Cannot do it here as we may have been long preempted. */
5006
5007#ifdef VBOX_STRICT
5008 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5009 pVmxTransient->uExitReason));
5010 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
5011 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5012 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5013 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5014 else
5015 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5016 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5017 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5018
5019 /* VMX control bits. */
5020 uint32_t u32Val;
5021 uint64_t u64Val;
5022 RTHCUINTREG uHCReg;
5023 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5024 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5025 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5026 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5027 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
5028 {
5029 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5030 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5031 }
5032 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5033 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5034 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5035 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5036 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5037 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5038 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5039 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5040 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5041 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5042 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5043 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5044 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5045 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5046 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5047 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5048 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5049 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5050 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5051 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5052 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5053 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5054 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5055 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5056 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5057 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5058 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5059 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5060 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5061 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5062 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5063 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5064 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5065 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5066 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5067 {
5068 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5069 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5070 }
5071
5072 /* Guest bits. */
5073 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5074 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
5075 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5076 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
5077 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5078 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
5079 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
5080 {
5081 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5082 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5083 }
5084
5085 /* Host bits. */
5086 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5087 Log4(("Host CR0 %#RHr\n", uHCReg));
5088 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5089 Log4(("Host CR3 %#RHr\n", uHCReg));
5090 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5091 Log4(("Host CR4 %#RHr\n", uHCReg));
5092
5093 RTGDTR HostGdtr;
5094 PCX86DESCHC pDesc;
5095 ASMGetGDTR(&HostGdtr);
5096 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5097 Log4(("Host CS %#08x\n", u32Val));
5098 if (u32Val < HostGdtr.cbGdt)
5099 {
5100 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5101 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5102 }
5103
5104 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5105 Log4(("Host DS %#08x\n", u32Val));
5106 if (u32Val < HostGdtr.cbGdt)
5107 {
5108 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5109 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5110 }
5111
5112 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5113 Log4(("Host ES %#08x\n", u32Val));
5114 if (u32Val < HostGdtr.cbGdt)
5115 {
5116 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5117 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5118 }
5119
5120 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5121 Log4(("Host FS %#08x\n", u32Val));
5122 if (u32Val < HostGdtr.cbGdt)
5123 {
5124 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5125 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5126 }
5127
5128 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5129 Log4(("Host GS %#08x\n", u32Val));
5130 if (u32Val < HostGdtr.cbGdt)
5131 {
5132 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5133 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5134 }
5135
5136 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5137 Log4(("Host SS %#08x\n", u32Val));
5138 if (u32Val < HostGdtr.cbGdt)
5139 {
5140 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5141 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5142 }
5143
5144 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5145 Log4(("Host TR %#08x\n", u32Val));
5146 if (u32Val < HostGdtr.cbGdt)
5147 {
5148 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5149 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5150 }
5151
5152 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5153 Log4(("Host TR Base %#RHv\n", uHCReg));
5154 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5155 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5156 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5157 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5158 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5159 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5160 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5161 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5162 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5163 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5164 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5165 Log4(("Host RSP %#RHv\n", uHCReg));
5166 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5167 Log4(("Host RIP %#RHv\n", uHCReg));
5168# if HC_ARCH_BITS == 64
5169 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5170 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5171 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5172 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5173 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5174 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5175# endif
5176#endif /* VBOX_STRICT */
5177 break;
5178 }
5179
5180 default:
5181 /* Impossible */
5182 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5183 break;
5184 }
5185}
5186
5187
5188#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5189#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5190# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5191#endif
5192#ifdef VBOX_STRICT
5193static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5194{
5195 switch (idxField)
5196 {
5197 case VMX_VMCS_GUEST_RIP:
5198 case VMX_VMCS_GUEST_RSP:
5199 case VMX_VMCS_GUEST_SYSENTER_EIP:
5200 case VMX_VMCS_GUEST_SYSENTER_ESP:
5201 case VMX_VMCS_GUEST_GDTR_BASE:
5202 case VMX_VMCS_GUEST_IDTR_BASE:
5203 case VMX_VMCS_GUEST_CS_BASE:
5204 case VMX_VMCS_GUEST_DS_BASE:
5205 case VMX_VMCS_GUEST_ES_BASE:
5206 case VMX_VMCS_GUEST_FS_BASE:
5207 case VMX_VMCS_GUEST_GS_BASE:
5208 case VMX_VMCS_GUEST_SS_BASE:
5209 case VMX_VMCS_GUEST_LDTR_BASE:
5210 case VMX_VMCS_GUEST_TR_BASE:
5211 case VMX_VMCS_GUEST_CR3:
5212 return true;
5213 }
5214 return false;
5215}
5216
5217static bool hmR0VmxIsValidReadField(uint32_t idxField)
5218{
5219 switch (idxField)
5220 {
5221 /* Read-only fields. */
5222 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5223 return true;
5224 }
5225 /* Remaining readable fields should also be writable. */
5226 return hmR0VmxIsValidWriteField(idxField);
5227}
5228#endif /* VBOX_STRICT */
5229
5230
5231/**
5232 * Executes the specified handler in 64-bit mode.
5233 *
5234 * @returns VBox status code (no informational status codes).
5235 * @param pVCpu The cross context virtual CPU structure.
5236 * @param enmOp The operation to perform.
5237 * @param cParams Number of parameters.
5238 * @param paParam Array of 32-bit parameters.
5239 */
5240VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
5241{
5242 PVM pVM = pVCpu->CTX_SUFF(pVM);
5243 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5244 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5245 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5246 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5247
5248#ifdef VBOX_STRICT
5249 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5250 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5251
5252 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5253 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5254#endif
5255
5256 /* Disable interrupts. */
5257 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5258
5259#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5260 RTCPUID idHostCpu = RTMpCpuId();
5261 CPUMR0SetLApic(pVCpu, idHostCpu);
5262#endif
5263
5264 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5265 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5266
5267 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5268 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5269 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5270
5271 /* Leave VMX Root Mode. */
5272 VMXDisable();
5273
5274 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5275
5276 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5277 CPUMSetHyperEIP(pVCpu, enmOp);
5278 for (int i = (int)cParams - 1; i >= 0; i--)
5279 CPUMPushHyper(pVCpu, paParam[i]);
5280
5281 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5282
5283 /* Call the switcher. */
5284 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
5285 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5286
5287 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5288 /* Make sure the VMX instructions don't cause #UD faults. */
5289 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5290
5291 /* Re-enter VMX Root Mode */
5292 int rc2 = VMXEnable(HCPhysCpuPage);
5293 if (RT_FAILURE(rc2))
5294 {
5295 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5296 ASMSetFlags(fOldEFlags);
5297 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5298 return rc2;
5299 }
5300
5301 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5302 AssertRC(rc2);
5303 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5304 Assert(!(ASMGetFlags() & X86_EFL_IF));
5305 ASMSetFlags(fOldEFlags);
5306 return rc;
5307}
5308
5309
5310/**
5311 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5312 * supporting 64-bit guests.
5313 *
5314 * @returns VBox status code.
5315 * @param fResume Whether to VMLAUNCH or VMRESUME.
5316 * @param pCtx Pointer to the guest-CPU context.
5317 * @param pCache Pointer to the VMCS cache.
5318 * @param pVM The cross context VM structure.
5319 * @param pVCpu The cross context virtual CPU structure.
5320 */
5321DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5322{
5323 NOREF(fResume);
5324
5325 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5326 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5327
5328#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5329 pCache->uPos = 1;
5330 pCache->interPD = PGMGetInterPaeCR3(pVM);
5331 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5332#endif
5333
5334#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5335 pCache->TestIn.HCPhysCpuPage = 0;
5336 pCache->TestIn.HCPhysVmcs = 0;
5337 pCache->TestIn.pCache = 0;
5338 pCache->TestOut.HCPhysVmcs = 0;
5339 pCache->TestOut.pCache = 0;
5340 pCache->TestOut.pCtx = 0;
5341 pCache->TestOut.eflags = 0;
5342#else
5343 NOREF(pCache);
5344#endif
5345
5346 uint32_t aParam[10];
5347 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5348 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5349 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5350 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5351 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5352 aParam[5] = 0;
5353 aParam[6] = VM_RC_ADDR(pVM, pVM);
5354 aParam[7] = 0;
5355 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5356 aParam[9] = 0;
5357
5358#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5359 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5360 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5361#endif
5362 int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5363
5364#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5365 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5366 Assert(pCtx->dr[4] == 10);
5367 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5368#endif
5369
5370#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5371 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5372 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5373 pVCpu->hm.s.vmx.HCPhysVmcs));
5374 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5375 pCache->TestOut.HCPhysVmcs));
5376 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5377 pCache->TestOut.pCache));
5378 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5379 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5380 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5381 pCache->TestOut.pCtx));
5382 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5383#endif
5384 NOREF(pCtx);
5385 return rc;
5386}
5387
5388
5389/**
5390 * Initialize the VMCS-Read cache.
5391 *
5392 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5393 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5394 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5395 * (those that have a 32-bit FULL & HIGH part).
5396 *
5397 * @returns VBox status code.
5398 * @param pVCpu The cross context virtual CPU structure.
5399 */
5400static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
5401{
5402#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5403 do { \
5404 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5405 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5406 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5407 ++cReadFields; \
5408 } while (0)
5409
5410 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5411 uint32_t cReadFields = 0;
5412
5413 /*
5414 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5415 * and serve to indicate exceptions to the rules.
5416 */
5417
5418 /* Guest-natural selector base fields. */
5419#if 0
5420 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5421 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5422 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5423#endif
5424 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5425 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5426 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5427 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5428 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5429 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5430 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5431 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5432 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5433 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5434 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5435 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5436#if 0
5437 /* Unused natural width guest-state fields. */
5438 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5439 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5440#endif
5441 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5442 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5443
5444 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5445 these 64-bit fields (using "FULL" and "HIGH" fields). */
5446#if 0
5447 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5448 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5449 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5450 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5451 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5452 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5453 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5454 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5455 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5456#endif
5457
5458 /* Natural width guest-state fields. */
5459 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5460 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_GUEST_LINEAR_ADDR);
5461
5462 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5463 {
5464 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5465 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5466 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5467 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5468 }
5469 else
5470 {
5471 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5472 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5473 }
5474
5475#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5476 return VINF_SUCCESS;
5477}
5478
5479
5480/**
5481 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5482 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5483 * darwin, running 64-bit guests).
5484 *
5485 * @returns VBox status code.
5486 * @param pVCpu The cross context virtual CPU structure.
5487 * @param idxField The VMCS field encoding.
5488 * @param u64Val 16, 32 or 64-bit value.
5489 */
5490VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5491{
5492 int rc;
5493 switch (idxField)
5494 {
5495 /*
5496 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5497 */
5498 /* 64-bit Control fields. */
5499 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5500 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5501 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5502 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5503 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5504 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5505 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5506 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5507 case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
5508 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5509 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5510 case VMX_VMCS64_CTRL_EPTP_FULL:
5511 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5512 /* 64-bit Guest-state fields. */
5513 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5514 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5515 case VMX_VMCS64_GUEST_PAT_FULL:
5516 case VMX_VMCS64_GUEST_EFER_FULL:
5517 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5518 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5519 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5520 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5521 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5522 /* 64-bit Host-state fields. */
5523 case VMX_VMCS64_HOST_PAT_FULL:
5524 case VMX_VMCS64_HOST_EFER_FULL:
5525 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5526 {
5527 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5528 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5529 break;
5530 }
5531
5532 /*
5533 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5534 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5535 */
5536 /* Natural-width Guest-state fields. */
5537 case VMX_VMCS_GUEST_CR3:
5538 case VMX_VMCS_GUEST_ES_BASE:
5539 case VMX_VMCS_GUEST_CS_BASE:
5540 case VMX_VMCS_GUEST_SS_BASE:
5541 case VMX_VMCS_GUEST_DS_BASE:
5542 case VMX_VMCS_GUEST_FS_BASE:
5543 case VMX_VMCS_GUEST_GS_BASE:
5544 case VMX_VMCS_GUEST_LDTR_BASE:
5545 case VMX_VMCS_GUEST_TR_BASE:
5546 case VMX_VMCS_GUEST_GDTR_BASE:
5547 case VMX_VMCS_GUEST_IDTR_BASE:
5548 case VMX_VMCS_GUEST_RSP:
5549 case VMX_VMCS_GUEST_RIP:
5550 case VMX_VMCS_GUEST_SYSENTER_ESP:
5551 case VMX_VMCS_GUEST_SYSENTER_EIP:
5552 {
5553 if (!(RT_HI_U32(u64Val)))
5554 {
5555 /* If this field is 64-bit, VT-x will zero out the top bits. */
5556 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5557 }
5558 else
5559 {
5560 /* Assert that only the 32->64 switcher case should ever come here. */
5561 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5562 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5563 }
5564 break;
5565 }
5566
5567 default:
5568 {
5569 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5570 rc = VERR_INVALID_PARAMETER;
5571 break;
5572 }
5573 }
5574 AssertRCReturn(rc, rc);
5575 return rc;
5576}
5577
5578
5579/**
5580 * Queue up a VMWRITE by using the VMCS write cache.
5581 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5582 *
5583 * @param pVCpu The cross context virtual CPU structure.
5584 * @param idxField The VMCS field encoding.
5585 * @param u64Val 16, 32 or 64-bit value.
5586 */
5587VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5588{
5589 AssertPtr(pVCpu);
5590 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5591
5592 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5593 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5594
5595 /* Make sure there are no duplicates. */
5596 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5597 {
5598 if (pCache->Write.aField[i] == idxField)
5599 {
5600 pCache->Write.aFieldVal[i] = u64Val;
5601 return VINF_SUCCESS;
5602 }
5603 }
5604
5605 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5606 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5607 pCache->Write.cValidEntries++;
5608 return VINF_SUCCESS;
5609}
5610#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5611
5612
5613/**
5614 * Sets up the usage of TSC-offsetting and updates the VMCS.
5615 *
5616 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5617 * VMX preemption timer.
5618 *
5619 * @returns VBox status code.
5620 * @param pVCpu The cross context virtual CPU structure.
5621 *
5622 * @remarks No-long-jump zone!!!
5623 */
5624static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5625{
5626 bool fOffsettedTsc;
5627 bool fParavirtTsc;
5628 PVM pVM = pVCpu->CTX_SUFF(pVM);
5629 uint64_t uTscOffset;
5630 if (pVM->hm.s.vmx.fUsePreemptTimer)
5631 {
5632 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
5633
5634 /* Make sure the returned values have sane upper and lower boundaries. */
5635 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5636 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5637 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5638 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5639
5640 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5641 int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
5642 AssertRC(rc);
5643 }
5644 else
5645 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
5646
5647 if (fParavirtTsc)
5648 {
5649 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5650 information before every VM-entry, hence disable it for performance sake. */
5651#if 0
5652 int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5653 AssertRC(rc);
5654#endif
5655 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5656 }
5657
5658 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
5659 if ( fOffsettedTsc
5660 && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5661 {
5662 if (pVCpu->hm.s.vmx.u64TscOffset != uTscOffset)
5663 {
5664 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
5665 AssertRC(rc);
5666 pVCpu->hm.s.vmx.u64TscOffset = uTscOffset;
5667 }
5668
5669 if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
5670 {
5671 uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
5672 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5673 AssertRC(rc);
5674 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5675 }
5676 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5677 }
5678 else
5679 {
5680 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5681 if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
5682 {
5683 uProcCtls |= VMX_PROC_CTLS_RDTSC_EXIT;
5684 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5685 AssertRC(rc);
5686 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5687 }
5688 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5689 }
5690}
5691
5692
5693/**
5694 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5695 * VM-exit interruption info type.
5696 *
5697 * @returns The IEM exception flags.
5698 * @param uVector The event vector.
5699 * @param uVmxVectorType The VMX event type.
5700 *
5701 * @remarks This function currently only constructs flags required for
5702 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5703 * and CR2 aspects of an exception are not included).
5704 */
5705static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5706{
5707 uint32_t fIemXcptFlags;
5708 switch (uVmxVectorType)
5709 {
5710 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5711 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5712 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5713 break;
5714
5715 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5716 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5717 break;
5718
5719 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5720 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5721 break;
5722
5723 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5724 {
5725 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5726 if (uVector == X86_XCPT_BP)
5727 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5728 else if (uVector == X86_XCPT_OF)
5729 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5730 else
5731 {
5732 fIemXcptFlags = 0;
5733 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5734 }
5735 break;
5736 }
5737
5738 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5739 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5740 break;
5741
5742 default:
5743 fIemXcptFlags = 0;
5744 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5745 break;
5746 }
5747 return fIemXcptFlags;
5748}
5749
5750
5751/**
5752 * Sets an event as a pending event to be injected into the guest.
5753 *
5754 * @param pVCpu The cross context virtual CPU structure.
5755 * @param u32IntInfo The VM-entry interruption-information field.
5756 * @param cbInstr The VM-entry instruction length in bytes (for software
5757 * interrupts, exceptions and privileged software
5758 * exceptions).
5759 * @param u32ErrCode The VM-entry exception error code.
5760 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5761 * page-fault.
5762 *
5763 * @remarks Statistics counter assumes this is a guest event being injected or
5764 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5765 * always incremented.
5766 */
5767DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5768 RTGCUINTPTR GCPtrFaultAddress)
5769{
5770 Assert(!pVCpu->hm.s.Event.fPending);
5771 pVCpu->hm.s.Event.fPending = true;
5772 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5773 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5774 pVCpu->hm.s.Event.cbInstr = cbInstr;
5775 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5776}
5777
5778
5779/**
5780 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5781 *
5782 * @param pVCpu The cross context virtual CPU structure.
5783 */
5784DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
5785{
5786 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
5787 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5788 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5789 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5790 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5791}
5792
5793
5794/**
5795 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
5796 *
5797 * @param pVCpu The cross context virtual CPU structure.
5798 */
5799DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
5800{
5801 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
5802 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5803 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
5804 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5805 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5806}
5807
5808
5809/**
5810 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
5811 *
5812 * @param pVCpu The cross context virtual CPU structure.
5813 */
5814DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
5815{
5816 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
5817 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5818 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
5819 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5820 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5821}
5822
5823
5824#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5825/**
5826 * Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
5827 *
5828 * @param pVCpu The cross context virtual CPU structure.
5829 * @param u32ErrCode The error code for the general-protection exception.
5830 */
5831DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
5832{
5833 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
5834 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5835 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5836 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5837 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrCode, 0 /* GCPtrFaultAddress */);
5838}
5839
5840
5841/**
5842 * Sets a stack (\#SS) exception as pending-for-injection into the VM.
5843 *
5844 * @param pVCpu The cross context virtual CPU structure.
5845 * @param u32ErrCode The error code for the stack exception.
5846 */
5847DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
5848{
5849 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
5850 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5851 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5852 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5853 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrCode, 0 /* GCPtrFaultAddress */);
5854}
5855
5856
5857/**
5858 * Decodes the memory operand of an instruction that caused a VM-exit.
5859 *
5860 * The VM-exit qualification field provides the displacement field for memory
5861 * operand instructions, if any.
5862 *
5863 * @returns Strict VBox status code (i.e. informational status codes too).
5864 * @retval VINF_SUCCESS if the operand was successfully decoded.
5865 * @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
5866 * operand.
5867 * @param pVCpu The cross context virtual CPU structure.
5868 * @param uExitInstrInfo The VM-exit instruction information field.
5869 * @param enmMemAccess The memory operand's access type (read or write).
5870 * @param GCPtrDisp The instruction displacement field, if any. For
5871 * RIP-relative addressing pass RIP + displacement here.
5872 * @param pGCPtrMem Where to store the effective destination memory address.
5873 */
5874static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
5875 PRTGCPTR pGCPtrMem)
5876{
5877 Assert(pGCPtrMem);
5878 Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
5879 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_EFER | CPUMCTX_EXTRN_CR0);
5880
5881 static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
5882 static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
5883 AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
5884
5885 VMXEXITINSTRINFO ExitInstrInfo;
5886 ExitInstrInfo.u = uExitInstrInfo;
5887 uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
5888 uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
5889 bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
5890 uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
5891 uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
5892 bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
5893 uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
5894 bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
5895 bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
5896
5897 /*
5898 * Validate instruction information.
5899 * This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
5900 */
5901 AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
5902 ("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
5903 AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
5904 ("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
5905 AssertLogRelMsgReturn(fIsMemOperand,
5906 ("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
5907
5908 /*
5909 * Compute the complete effective address.
5910 *
5911 * See AMD instruction spec. 1.4.2 "SIB Byte Format"
5912 * See AMD spec. 4.5.2 "Segment Registers".
5913 */
5914 RTGCPTR GCPtrMem = GCPtrDisp;
5915 if (fBaseRegValid)
5916 GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
5917 if (fIdxRegValid)
5918 GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
5919
5920 RTGCPTR const GCPtrOff = GCPtrMem;
5921 if ( !fIsLongMode
5922 || iSegReg >= X86_SREG_FS)
5923 GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
5924 GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
5925
5926 /*
5927 * Validate effective address.
5928 * See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
5929 */
5930 uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
5931 Assert(cbAccess > 0);
5932 if (fIsLongMode)
5933 {
5934 if (X86_IS_CANONICAL(GCPtrMem))
5935 {
5936 *pGCPtrMem = GCPtrMem;
5937 return VINF_SUCCESS;
5938 }
5939
5940 /** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
5941 * "Data Limit Checks in 64-bit Mode". */
5942 Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
5943 hmR0VmxSetPendingXcptGP(pVCpu, 0);
5944 return VINF_HM_PENDING_XCPT;
5945 }
5946
5947 /*
5948 * This is a watered down version of iemMemApplySegment().
5949 * Parts that are not applicable for VMX instructions like real-or-v8086 mode
5950 * and segment CPL/DPL checks are skipped.
5951 */
5952 RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
5953 RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
5954 PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
5955
5956 /* Check if the segment is present and usable. */
5957 if ( pSel->Attr.n.u1Present
5958 && !pSel->Attr.n.u1Unusable)
5959 {
5960 Assert(pSel->Attr.n.u1DescType);
5961 if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
5962 {
5963 /* Check permissions for the data segment. */
5964 if ( enmMemAccess == VMXMEMACCESS_WRITE
5965 && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
5966 {
5967 Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
5968 hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
5969 return VINF_HM_PENDING_XCPT;
5970 }
5971
5972 /* Check limits if it's a normal data segment. */
5973 if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
5974 {
5975 if ( GCPtrFirst32 > pSel->u32Limit
5976 || GCPtrLast32 > pSel->u32Limit)
5977 {
5978 Log4Func(("Data segment limit exceeded."
5979 "iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
5980 GCPtrLast32, pSel->u32Limit));
5981 if (iSegReg == X86_SREG_SS)
5982 hmR0VmxSetPendingXcptSS(pVCpu, 0);
5983 else
5984 hmR0VmxSetPendingXcptGP(pVCpu, 0);
5985 return VINF_HM_PENDING_XCPT;
5986 }
5987 }
5988 else
5989 {
5990 /* Check limits if it's an expand-down data segment.
5991 Note! The upper boundary is defined by the B bit, not the G bit! */
5992 if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
5993 || GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
5994 {
5995 Log4Func(("Expand-down data segment limit exceeded."
5996 "iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
5997 GCPtrLast32, pSel->u32Limit));
5998 if (iSegReg == X86_SREG_SS)
5999 hmR0VmxSetPendingXcptSS(pVCpu, 0);
6000 else
6001 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6002 return VINF_HM_PENDING_XCPT;
6003 }
6004 }
6005 }
6006 else
6007 {
6008 /* Check permissions for the code segment. */
6009 if ( enmMemAccess == VMXMEMACCESS_WRITE
6010 || ( enmMemAccess == VMXMEMACCESS_READ
6011 && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
6012 {
6013 Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
6014 Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
6015 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6016 return VINF_HM_PENDING_XCPT;
6017 }
6018
6019 /* Check limits for the code segment (normal/expand-down not applicable for code segments). */
6020 if ( GCPtrFirst32 > pSel->u32Limit
6021 || GCPtrLast32 > pSel->u32Limit)
6022 {
6023 Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
6024 GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
6025 if (iSegReg == X86_SREG_SS)
6026 hmR0VmxSetPendingXcptSS(pVCpu, 0);
6027 else
6028 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6029 return VINF_HM_PENDING_XCPT;
6030 }
6031 }
6032 }
6033 else
6034 {
6035 Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6036 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6037 return VINF_HM_PENDING_XCPT;
6038 }
6039
6040 *pGCPtrMem = GCPtrMem;
6041 return VINF_SUCCESS;
6042}
6043
6044
6045/**
6046 * Perform the relevant VMX instruction checks for VM-exits that occurred due to the
6047 * guest attempting to execute a VMX instruction.
6048 *
6049 * @returns Strict VBox status code (i.e. informational status codes too).
6050 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
6051 * @retval VINF_HM_PENDING_XCPT if an exception was raised.
6052 *
6053 * @param pVCpu The cross context virtual CPU structure.
6054 * @param uExitReason The VM-exit reason.
6055 *
6056 * @todo NstVmx: Document other error codes when VM-exit is implemented.
6057 * @remarks No-long-jump zone!!!
6058 */
6059static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
6060{
6061 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS
6062 | CPUMCTX_EXTRN_HWVIRT);
6063
6064 if ( CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx)
6065 || ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6066 && !CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx)))
6067 {
6068 Log4Func(("In real/v86-mode or long-mode outside 64-bit code segment -> #UD\n"));
6069 hmR0VmxSetPendingXcptUD(pVCpu);
6070 return VINF_HM_PENDING_XCPT;
6071 }
6072
6073 if (uExitReason == VMX_EXIT_VMXON)
6074 {
6075 /*
6076 * We check CR4.VMXE because it is required to be always set while in VMX operation
6077 * by physical CPUs and our CR4 read shadow is only consulted when executing specific
6078 * instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
6079 * otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
6080 */
6081 if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
6082 {
6083 Log4Func(("CR4.VMXE is not set -> #UD\n"));
6084 hmR0VmxSetPendingXcptUD(pVCpu);
6085 return VINF_HM_PENDING_XCPT;
6086 }
6087 }
6088 else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
6089 {
6090 /*
6091 * The guest has not entered VMX operation but attempted to execute a VMX instruction
6092 * (other than VMXON), we need to raise a #UD.
6093 */
6094 Log4Func(("Not in VMX root mode -> #UD\n"));
6095 hmR0VmxSetPendingXcptUD(pVCpu);
6096 return VINF_HM_PENDING_XCPT;
6097 }
6098
6099 if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
6100 {
6101 /*
6102 * The nested-guest attempted to execute a VMX instruction, cause a VM-exit and let
6103 * the guest hypervisor deal with it.
6104 */
6105 /** @todo NSTVMX: Trigger a VM-exit */
6106 }
6107
6108 /*
6109 * VMX instructions require CPL 0 except in VMX non-root mode where the VM-exit intercept
6110 * (above) takes preceedence over the CPL check.
6111 */
6112 if (CPUMGetGuestCPL(pVCpu) > 0)
6113 {
6114 Log4Func(("CPL > 0 -> #GP(0)\n"));
6115 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6116 return VINF_HM_PENDING_XCPT;
6117 }
6118
6119 return VINF_SUCCESS;
6120}
6121
6122#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
6123
6124
6125/**
6126 * Handle a condition that occurred while delivering an event through the guest
6127 * IDT.
6128 *
6129 * @returns Strict VBox status code (i.e. informational status codes too).
6130 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
6131 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
6132 * to continue execution of the guest which will delivery the \#DF.
6133 * @retval VINF_EM_RESET if we detected a triple-fault condition.
6134 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
6135 *
6136 * @param pVCpu The cross context virtual CPU structure.
6137 * @param pVmxTransient Pointer to the VMX transient structure.
6138 *
6139 * @remarks No-long-jump zone!!!
6140 */
6141static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6142{
6143 uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
6144
6145 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
6146 rc2 |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
6147 AssertRCReturn(rc2, rc2);
6148
6149 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
6150 if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6151 {
6152 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
6153 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
6154
6155 /*
6156 * If the event was a software interrupt (generated with INT n) or a software exception
6157 * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
6158 * can handle the VM-exit and continue guest execution which will re-execute the
6159 * instruction rather than re-injecting the exception, as that can cause premature
6160 * trips to ring-3 before injection and involve TRPM which currently has no way of
6161 * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
6162 * the problem).
6163 */
6164 IEMXCPTRAISE enmRaise;
6165 IEMXCPTRAISEINFO fRaiseInfo;
6166 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6167 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6168 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6169 {
6170 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
6171 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6172 }
6173 else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
6174 {
6175 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
6176 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
6177 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
6178 /** @todo Make AssertMsgReturn as just AssertMsg later. */
6179 AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
6180 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
6181 uExitVectorType), VERR_VMX_IPE_5);
6182
6183 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
6184
6185 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
6186 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
6187 {
6188 pVmxTransient->fVectoringPF = true;
6189 enmRaise = IEMXCPTRAISE_PREV_EVENT;
6190 }
6191 }
6192 else
6193 {
6194 /*
6195 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
6196 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
6197 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
6198 */
6199 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6200 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6201 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
6202 enmRaise = IEMXCPTRAISE_PREV_EVENT;
6203 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6204 }
6205
6206 /*
6207 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
6208 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
6209 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
6210 * subsequent VM-entry would fail.
6211 *
6212 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
6213 */
6214 if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS)
6215 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6216 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
6217 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
6218 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
6219 {
6220 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6221 }
6222
6223 switch (enmRaise)
6224 {
6225 case IEMXCPTRAISE_CURRENT_XCPT:
6226 {
6227 Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
6228 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
6229 Assert(rcStrict == VINF_SUCCESS);
6230 break;
6231 }
6232
6233 case IEMXCPTRAISE_PREV_EVENT:
6234 {
6235 uint32_t u32ErrCode;
6236 if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
6237 {
6238 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6239 AssertRCReturn(rc2, rc2);
6240 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6241 }
6242 else
6243 u32ErrCode = 0;
6244
6245 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
6246 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6247 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6248 0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
6249
6250 Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
6251 pVCpu->hm.s.Event.u32ErrCode));
6252 Assert(rcStrict == VINF_SUCCESS);
6253 break;
6254 }
6255
6256 case IEMXCPTRAISE_REEXEC_INSTR:
6257 Assert(rcStrict == VINF_SUCCESS);
6258 break;
6259
6260 case IEMXCPTRAISE_DOUBLE_FAULT:
6261 {
6262 /*
6263 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
6264 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
6265 */
6266 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6267 {
6268 pVmxTransient->fVectoringDoublePF = true;
6269 Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
6270 pVCpu->cpum.GstCtx.cr2));
6271 rcStrict = VINF_SUCCESS;
6272 }
6273 else
6274 {
6275 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6276 hmR0VmxSetPendingXcptDF(pVCpu);
6277 Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
6278 uIdtVector, uExitVector));
6279 rcStrict = VINF_HM_DOUBLE_FAULT;
6280 }
6281 break;
6282 }
6283
6284 case IEMXCPTRAISE_TRIPLE_FAULT:
6285 {
6286 Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
6287 rcStrict = VINF_EM_RESET;
6288 break;
6289 }
6290
6291 case IEMXCPTRAISE_CPU_HANG:
6292 {
6293 Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
6294 rcStrict = VERR_EM_GUEST_CPU_HANG;
6295 break;
6296 }
6297
6298 default:
6299 {
6300 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6301 rcStrict = VERR_VMX_IPE_2;
6302 break;
6303 }
6304 }
6305 }
6306 else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6307 && VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6308 && uExitVector != X86_XCPT_DF
6309 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
6310 {
6311 /*
6312 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6313 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6314 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6315 */
6316 if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
6317 {
6318 Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
6319 VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6320 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6321 }
6322 }
6323
6324 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
6325 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
6326 return rcStrict;
6327}
6328
6329
6330/**
6331 * Imports a guest segment register from the current VMCS into
6332 * the guest-CPU context.
6333 *
6334 * @returns VBox status code.
6335 * @param pVCpu The cross context virtual CPU structure.
6336 * @param idxSel Index of the selector in the VMCS.
6337 * @param idxLimit Index of the segment limit in the VMCS.
6338 * @param idxBase Index of the segment base in the VMCS.
6339 * @param idxAccess Index of the access rights of the segment in the VMCS.
6340 * @param pSelReg Pointer to the segment selector.
6341 *
6342 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6343 * do not log!
6344 *
6345 * @remarks Never call this function directly!!! Use the
6346 * HMVMX_IMPORT_SREG() macro as that takes care
6347 * of whether to read from the VMCS cache or not.
6348 */
6349static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6350 PCPUMSELREG pSelReg)
6351{
6352 NOREF(pVCpu);
6353
6354 uint32_t u32Sel;
6355 uint32_t u32Limit;
6356 uint32_t u32Attr;
6357 uint64_t u64Base;
6358 int rc = VMXReadVmcs32(idxSel, &u32Sel);
6359 rc |= VMXReadVmcs32(idxLimit, &u32Limit);
6360 rc |= VMXReadVmcs32(idxAccess, &u32Attr);
6361 rc |= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
6362 AssertRCReturn(rc, rc);
6363
6364 pSelReg->Sel = (uint16_t)u32Sel;
6365 pSelReg->ValidSel = (uint16_t)u32Sel;
6366 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6367 pSelReg->u32Limit = u32Limit;
6368 pSelReg->u64Base = u64Base;
6369 pSelReg->Attr.u = u32Attr;
6370
6371 /*
6372 * If VT-x marks the segment as unusable, most other bits remain undefined:
6373 * - For CS the L, D and G bits have meaning.
6374 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6375 * - For the remaining data segments no bits are defined.
6376 *
6377 * The present bit and the unusable bit has been observed to be set at the
6378 * same time (the selector was supposed to be invalid as we started executing
6379 * a V8086 interrupt in ring-0).
6380 *
6381 * What should be important for the rest of the VBox code, is that the P bit is
6382 * cleared. Some of the other VBox code recognizes the unusable bit, but
6383 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6384 * safe side here, we'll strip off P and other bits we don't care about. If
6385 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6386 *
6387 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6388 */
6389 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6390 {
6391 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6392
6393 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6394 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6395 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6396#ifdef VBOX_STRICT
6397 VMMRZCallRing3Disable(pVCpu);
6398 Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
6399# ifdef DEBUG_bird
6400 AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
6401 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6402 idxSel, u32Sel, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6403# endif
6404 VMMRZCallRing3Enable(pVCpu);
6405#endif
6406 }
6407 return VINF_SUCCESS;
6408}
6409
6410
6411/**
6412 * Imports the guest RIP from the VMCS back into the guest-CPU context.
6413 *
6414 * @returns VBox status code.
6415 * @param pVCpu The cross context virtual CPU structure.
6416 *
6417 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6418 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6419 * instead!!!
6420 */
6421DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6422{
6423 uint64_t u64Val;
6424 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6425 if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6426 {
6427 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6428 if (RT_SUCCESS(rc))
6429 {
6430 pCtx->rip = u64Val;
6431 EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
6432 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6433 }
6434 return rc;
6435 }
6436 return VINF_SUCCESS;
6437}
6438
6439
6440/**
6441 * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6442 *
6443 * @returns VBox status code.
6444 * @param pVCpu The cross context virtual CPU structure.
6445 *
6446 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6447 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6448 * instead!!!
6449 */
6450DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6451{
6452 uint32_t u32Val;
6453 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6454 if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6455 {
6456 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6457 if (RT_SUCCESS(rc))
6458 {
6459 pCtx->eflags.u32 = u32Val;
6460
6461 /* Restore eflags for real-on-v86-mode hack. */
6462 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6463 {
6464 pCtx->eflags.Bits.u1VM = 0;
6465 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6466 }
6467 }
6468 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6469 return rc;
6470 }
6471 return VINF_SUCCESS;
6472}
6473
6474
6475/**
6476 * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6477 * context.
6478 *
6479 * @returns VBox status code.
6480 * @param pVCpu The cross context virtual CPU structure.
6481 *
6482 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6483 * do not log!
6484 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6485 * instead!!!
6486 */
6487DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6488{
6489 uint32_t u32Val;
6490 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6491 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
6492 if (RT_SUCCESS(rc))
6493 {
6494 /*
6495 * We additionally have a requirement to import RIP, RFLAGS depending on whether we
6496 * might need them in hmR0VmxEvaluatePendingEvent().
6497 */
6498 if (!u32Val)
6499 {
6500 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6501 {
6502 rc = hmR0VmxImportGuestRip(pVCpu);
6503 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6504 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6505 }
6506
6507 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
6508 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6509 }
6510 else
6511 {
6512 rc = hmR0VmxImportGuestRip(pVCpu);
6513 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6514
6515 if (u32Val & ( VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
6516 | VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
6517 {
6518 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6519 }
6520 else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6521 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6522
6523 if (u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
6524 {
6525 if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
6526 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6527 }
6528 else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
6529 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6530 }
6531 }
6532 return rc;
6533}
6534
6535
6536/**
6537 * Worker for VMXR0ImportStateOnDemand.
6538 *
6539 * @returns VBox status code.
6540 * @param pVCpu The cross context virtual CPU structure.
6541 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6542 */
6543static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6544{
6545#define VMXLOCAL_BREAK_RC(a_rc) \
6546 if (RT_FAILURE(a_rc)) \
6547 break
6548
6549 int rc = VINF_SUCCESS;
6550 PVM pVM = pVCpu->CTX_SUFF(pVM);
6551 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6552 uint64_t u64Val;
6553 uint32_t u32Val;
6554
6555 Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6556 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6557
6558 /*
6559 * We disable interrupts to make the updating of the state and in particular
6560 * the fExtrn modification atomic wrt to preemption hooks.
6561 */
6562 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6563
6564 fWhat &= pCtx->fExtrn;
6565 if (fWhat)
6566 {
6567 do
6568 {
6569 if (fWhat & CPUMCTX_EXTRN_RIP)
6570 {
6571 rc = hmR0VmxImportGuestRip(pVCpu);
6572 VMXLOCAL_BREAK_RC(rc);
6573 }
6574
6575 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6576 {
6577 rc = hmR0VmxImportGuestRFlags(pVCpu);
6578 VMXLOCAL_BREAK_RC(rc);
6579 }
6580
6581 if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6582 {
6583 rc = hmR0VmxImportGuestIntrState(pVCpu);
6584 VMXLOCAL_BREAK_RC(rc);
6585 }
6586
6587 if (fWhat & CPUMCTX_EXTRN_RSP)
6588 {
6589 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6590 VMXLOCAL_BREAK_RC(rc);
6591 pCtx->rsp = u64Val;
6592 }
6593
6594 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6595 {
6596 if (fWhat & CPUMCTX_EXTRN_CS)
6597 {
6598 rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6599 rc |= hmR0VmxImportGuestRip(pVCpu);
6600 VMXLOCAL_BREAK_RC(rc);
6601 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6602 pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6603 EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true);
6604 }
6605 if (fWhat & CPUMCTX_EXTRN_SS)
6606 {
6607 rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6608 VMXLOCAL_BREAK_RC(rc);
6609 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6610 pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6611 }
6612 if (fWhat & CPUMCTX_EXTRN_DS)
6613 {
6614 rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6615 VMXLOCAL_BREAK_RC(rc);
6616 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6617 pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6618 }
6619 if (fWhat & CPUMCTX_EXTRN_ES)
6620 {
6621 rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6622 VMXLOCAL_BREAK_RC(rc);
6623 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6624 pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6625 }
6626 if (fWhat & CPUMCTX_EXTRN_FS)
6627 {
6628 rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6629 VMXLOCAL_BREAK_RC(rc);
6630 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6631 pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6632 }
6633 if (fWhat & CPUMCTX_EXTRN_GS)
6634 {
6635 rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6636 VMXLOCAL_BREAK_RC(rc);
6637 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6638 pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6639 }
6640 }
6641
6642 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6643 {
6644 if (fWhat & CPUMCTX_EXTRN_LDTR)
6645 {
6646 rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6647 VMXLOCAL_BREAK_RC(rc);
6648 }
6649
6650 if (fWhat & CPUMCTX_EXTRN_GDTR)
6651 {
6652 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6653 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6654 VMXLOCAL_BREAK_RC(rc);
6655 pCtx->gdtr.pGdt = u64Val;
6656 pCtx->gdtr.cbGdt = u32Val;
6657 }
6658
6659 /* Guest IDTR. */
6660 if (fWhat & CPUMCTX_EXTRN_IDTR)
6661 {
6662 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6663 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6664 VMXLOCAL_BREAK_RC(rc);
6665 pCtx->idtr.pIdt = u64Val;
6666 pCtx->idtr.cbIdt = u32Val;
6667 }
6668
6669 /* Guest TR. */
6670 if (fWhat & CPUMCTX_EXTRN_TR)
6671 {
6672 /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6673 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6674 {
6675 rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6676 VMXLOCAL_BREAK_RC(rc);
6677 }
6678 }
6679 }
6680
6681 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6682 {
6683 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6684 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6685 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6686 pCtx->SysEnter.cs = u32Val;
6687 VMXLOCAL_BREAK_RC(rc);
6688 }
6689
6690#if HC_ARCH_BITS == 64
6691 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6692 {
6693 if ( pVM->hm.s.fAllow64BitGuests
6694 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6695 pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6696 }
6697
6698 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6699 {
6700 if ( pVM->hm.s.fAllow64BitGuests
6701 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6702 {
6703 pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6704 pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6705 pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6706 }
6707 }
6708#endif
6709
6710 if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
6711#if HC_ARCH_BITS == 32
6712 || (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS))
6713#endif
6714 )
6715 {
6716 PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6717 uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6718 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6719 {
6720 switch (pMsr->u32Msr)
6721 {
6722#if HC_ARCH_BITS == 32
6723 case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6724 case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6725 case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6726 case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6727#endif
6728 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6729 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6730 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit */ break;
6731 default:
6732 {
6733 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6734 ASMSetFlags(fEFlags);
6735 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6736 cMsrs));
6737 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6738 }
6739 }
6740 }
6741 }
6742
6743 if (fWhat & CPUMCTX_EXTRN_DR7)
6744 {
6745 if (!pVCpu->hm.s.fUsingHyperDR7)
6746 {
6747 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6748 rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6749 VMXLOCAL_BREAK_RC(rc);
6750 pCtx->dr[7] = u32Val;
6751 }
6752 }
6753
6754 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6755 {
6756 uint32_t u32Shadow;
6757 if (fWhat & CPUMCTX_EXTRN_CR0)
6758 {
6759 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6760 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6761 VMXLOCAL_BREAK_RC(rc);
6762 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr0Mask)
6763 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr0Mask);
6764 VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6765 CPUMSetGuestCR0(pVCpu, u32Val);
6766 VMMRZCallRing3Enable(pVCpu);
6767 }
6768
6769 if (fWhat & CPUMCTX_EXTRN_CR4)
6770 {
6771 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6772 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6773 VMXLOCAL_BREAK_RC(rc);
6774 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr4Mask)
6775 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr4Mask);
6776 CPUMSetGuestCR4(pVCpu, u32Val);
6777 }
6778
6779 if (fWhat & CPUMCTX_EXTRN_CR3)
6780 {
6781 /* CR0.PG bit changes are always intercepted, so it's up to date. */
6782 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6783 || ( pVM->hm.s.fNestedPaging
6784 && CPUMIsGuestPagingEnabledEx(pCtx)))
6785 {
6786 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6787 if (pCtx->cr3 != u64Val)
6788 {
6789 CPUMSetGuestCR3(pVCpu, u64Val);
6790 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6791 }
6792
6793 /* If the guest is in PAE mode, sync back the PDPE's into the guest state.
6794 Note: CR4.PAE, CR0.PG, EFER bit changes are always intercepted, so they're up to date. */
6795 if (CPUMIsGuestInPAEModeEx(pCtx))
6796 {
6797 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6798 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6799 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6800 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6801 VMXLOCAL_BREAK_RC(rc);
6802 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6803 }
6804 }
6805 }
6806 }
6807 } while (0);
6808
6809 if (RT_SUCCESS(rc))
6810 {
6811 /* Update fExtrn. */
6812 pCtx->fExtrn &= ~fWhat;
6813
6814 /* If everything has been imported, clear the HM keeper bit. */
6815 if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6816 {
6817 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6818 Assert(!pCtx->fExtrn);
6819 }
6820 }
6821 }
6822 else
6823 AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6824
6825 ASMSetFlags(fEFlags);
6826
6827 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6828
6829 /*
6830 * Honor any pending CR3 updates.
6831 *
6832 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6833 * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6834 * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6835 *
6836 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6837 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6838 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6839 * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6840 *
6841 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6842 */
6843 if (VMMRZCallRing3IsEnabled(pVCpu))
6844 {
6845 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6846 {
6847 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
6848 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6849 }
6850
6851 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6852 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6853
6854 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6855 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6856 }
6857
6858 return VINF_SUCCESS;
6859#undef VMXLOCAL_BREAK_RC
6860}
6861
6862
6863/**
6864 * Saves the guest state from the VMCS into the guest-CPU context.
6865 *
6866 * @returns VBox status code.
6867 * @param pVCpu The cross context virtual CPU structure.
6868 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6869 */
6870VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6871{
6872 return hmR0VmxImportGuestState(pVCpu, fWhat);
6873}
6874
6875
6876/**
6877 * Check per-VM and per-VCPU force flag actions that require us to go back to
6878 * ring-3 for one reason or another.
6879 *
6880 * @returns Strict VBox status code (i.e. informational status codes too)
6881 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6882 * ring-3.
6883 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6884 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6885 * interrupts)
6886 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6887 * all EMTs to be in ring-3.
6888 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6889 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6890 * to the EM loop.
6891 *
6892 * @param pVCpu The cross context virtual CPU structure.
6893 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6894 */
6895static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
6896{
6897 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6898
6899 /*
6900 * Anything pending? Should be more likely than not if we're doing a good job.
6901 */
6902 PVM pVM = pVCpu->CTX_SUFF(pVM);
6903 if ( !fStepping
6904 ? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6905 && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6906 : !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6907 && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6908 return VINF_SUCCESS;
6909
6910 /* Pending PGM C3 sync. */
6911 if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6912 {
6913 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6914 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
6915 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
6916 VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6917 if (rcStrict2 != VINF_SUCCESS)
6918 {
6919 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6920 Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6921 return rcStrict2;
6922 }
6923 }
6924
6925 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6926 if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
6927 || VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6928 {
6929 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6930 int rc2 = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
6931 Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6932 return rc2;
6933 }
6934
6935 /* Pending VM request packets, such as hardware interrupts. */
6936 if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
6937 || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
6938 {
6939 Log4Func(("Pending VM request forcing us back to ring-3\n"));
6940 return VINF_EM_PENDING_REQUEST;
6941 }
6942
6943 /* Pending PGM pool flushes. */
6944 if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6945 {
6946 Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6947 return VINF_PGM_POOL_FLUSH_PENDING;
6948 }
6949
6950 /* Pending DMA requests. */
6951 if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
6952 {
6953 Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6954 return VINF_EM_RAW_TO_R3;
6955 }
6956
6957 return VINF_SUCCESS;
6958}
6959
6960
6961/**
6962 * Converts any TRPM trap into a pending HM event. This is typically used when
6963 * entering from ring-3 (not longjmp returns).
6964 *
6965 * @param pVCpu The cross context virtual CPU structure.
6966 */
6967static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6968{
6969 Assert(TRPMHasTrap(pVCpu));
6970 Assert(!pVCpu->hm.s.Event.fPending);
6971
6972 uint8_t uVector;
6973 TRPMEVENT enmTrpmEvent;
6974 RTGCUINT uErrCode;
6975 RTGCUINTPTR GCPtrFaultAddress;
6976 uint8_t cbInstr;
6977
6978 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6979 AssertRC(rc);
6980
6981 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6982 uint32_t u32IntInfo = uVector | VMX_EXIT_INT_INFO_VALID;
6983 if (enmTrpmEvent == TRPM_TRAP)
6984 {
6985 switch (uVector)
6986 {
6987 case X86_XCPT_NMI:
6988 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6989 break;
6990
6991 case X86_XCPT_BP:
6992 case X86_XCPT_OF:
6993 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6994 break;
6995
6996 case X86_XCPT_PF:
6997 case X86_XCPT_DF:
6998 case X86_XCPT_TS:
6999 case X86_XCPT_NP:
7000 case X86_XCPT_SS:
7001 case X86_XCPT_GP:
7002 case X86_XCPT_AC:
7003 u32IntInfo |= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7004 RT_FALL_THRU();
7005 default:
7006 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7007 break;
7008 }
7009 }
7010 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7011 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7012 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7013 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7014 else
7015 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7016
7017 rc = TRPMResetTrap(pVCpu);
7018 AssertRC(rc);
7019 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7020 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7021
7022 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7023}
7024
7025
7026/**
7027 * Converts the pending HM event into a TRPM trap.
7028 *
7029 * @param pVCpu The cross context virtual CPU structure.
7030 */
7031static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7032{
7033 Assert(pVCpu->hm.s.Event.fPending);
7034
7035 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7036 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7037 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
7038 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7039
7040 /* If a trap was already pending, we did something wrong! */
7041 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7042
7043 TRPMEVENT enmTrapType;
7044 switch (uVectorType)
7045 {
7046 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7047 enmTrapType = TRPM_HARDWARE_INT;
7048 break;
7049
7050 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7051 enmTrapType = TRPM_SOFTWARE_INT;
7052 break;
7053
7054 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7055 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7056 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7057 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7058 enmTrapType = TRPM_TRAP;
7059 break;
7060
7061 default:
7062 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7063 enmTrapType = TRPM_32BIT_HACK;
7064 break;
7065 }
7066
7067 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7068
7069 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7070 AssertRC(rc);
7071
7072 if (fErrorCodeValid)
7073 TRPMSetErrorCode(pVCpu, uErrorCode);
7074
7075 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7076 && uVector == X86_XCPT_PF)
7077 {
7078 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7079 }
7080 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7081 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7082 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7083 {
7084 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7085 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
7086 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7087 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7088 }
7089
7090 /* Clear any pending events from the VMCS. */
7091 VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7092 VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, 0);
7093
7094 /* We're now done converting the pending event. */
7095 pVCpu->hm.s.Event.fPending = false;
7096}
7097
7098
7099/**
7100 * Does the necessary state syncing before returning to ring-3 for any reason
7101 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7102 *
7103 * @returns VBox status code.
7104 * @param pVCpu The cross context virtual CPU structure.
7105 * @param fImportState Whether to import the guest state from the VMCS back
7106 * to the guest-CPU context.
7107 *
7108 * @remarks No-long-jmp zone!!!
7109 */
7110static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
7111{
7112 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7113 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7114
7115 RTCPUID idCpu = RTMpCpuId();
7116 Log4Func(("HostCpuId=%u\n", idCpu));
7117
7118 /*
7119 * !!! IMPORTANT !!!
7120 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7121 */
7122
7123 /* Save the guest state if necessary. */
7124 if (fImportState)
7125 {
7126 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
7127 AssertRCReturn(rc, rc);
7128 }
7129
7130 /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
7131 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7132 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
7133
7134 /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
7135#ifdef VBOX_STRICT
7136 if (CPUMIsHyperDebugStateActive(pVCpu))
7137 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
7138#endif
7139 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7140 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7141 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7142
7143#if HC_ARCH_BITS == 64
7144 /* Restore host-state bits that VT-x only restores partially. */
7145 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7146 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7147 {
7148 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7149 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7150 }
7151 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7152#endif
7153
7154 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7155 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7156 {
7157 /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
7158 if (!fImportState)
7159 {
7160 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS);
7161 AssertRCReturn(rc, rc);
7162 }
7163 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7164 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7165 }
7166 else
7167 pVCpu->hm.s.vmx.fLazyMsrs = 0;
7168
7169 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7170 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7171
7172 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7173 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
7174 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
7175 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
7176 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
7177 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7178 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7179 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7180 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7181
7182 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7183
7184 /** @todo This partially defeats the purpose of having preemption hooks.
7185 * The problem is, deregistering the hooks should be moved to a place that
7186 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7187 * context.
7188 */
7189 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7190 {
7191 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7192 AssertRCReturn(rc, rc);
7193
7194 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7195 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7196 }
7197 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7198 NOREF(idCpu);
7199
7200 return VINF_SUCCESS;
7201}
7202
7203
7204/**
7205 * Leaves the VT-x session.
7206 *
7207 * @returns VBox status code.
7208 * @param pVCpu The cross context virtual CPU structure.
7209 *
7210 * @remarks No-long-jmp zone!!!
7211 */
7212static int hmR0VmxLeaveSession(PVMCPU pVCpu)
7213{
7214 HM_DISABLE_PREEMPT(pVCpu);
7215 HMVMX_ASSERT_CPU_SAFE(pVCpu);
7216 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7217 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7218
7219 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7220 and done this from the VMXR0ThreadCtxCallback(). */
7221 if (!pVCpu->hm.s.fLeaveDone)
7222 {
7223 int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
7224 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7225 pVCpu->hm.s.fLeaveDone = true;
7226 }
7227 Assert(!pVCpu->cpum.GstCtx.fExtrn);
7228
7229 /*
7230 * !!! IMPORTANT !!!
7231 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7232 */
7233
7234 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7235 /** @todo Deregistering here means we need to VMCLEAR always
7236 * (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
7237 * for calling VMMR0ThreadCtxHookDisable here! */
7238 VMMR0ThreadCtxHookDisable(pVCpu);
7239
7240 /* Leave HM context. This takes care of local init (term). */
7241 int rc = HMR0LeaveCpu(pVCpu);
7242
7243 HM_RESTORE_PREEMPT();
7244 return rc;
7245}
7246
7247
7248/**
7249 * Does the necessary state syncing before doing a longjmp to ring-3.
7250 *
7251 * @returns VBox status code.
7252 * @param pVCpu The cross context virtual CPU structure.
7253 *
7254 * @remarks No-long-jmp zone!!!
7255 */
7256DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
7257{
7258 return hmR0VmxLeaveSession(pVCpu);
7259}
7260
7261
7262/**
7263 * Take necessary actions before going back to ring-3.
7264 *
7265 * An action requires us to go back to ring-3. This function does the necessary
7266 * steps before we can safely return to ring-3. This is not the same as longjmps
7267 * to ring-3, this is voluntary and prepares the guest so it may continue
7268 * executing outside HM (recompiler/IEM).
7269 *
7270 * @returns VBox status code.
7271 * @param pVCpu The cross context virtual CPU structure.
7272 * @param rcExit The reason for exiting to ring-3. Can be
7273 * VINF_VMM_UNKNOWN_RING3_CALL.
7274 */
7275static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
7276{
7277 Assert(pVCpu);
7278 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7279
7280 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7281 {
7282 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7283 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7284 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7285 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7286 }
7287
7288 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7289 VMMRZCallRing3Disable(pVCpu);
7290 Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
7291
7292 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7293 if (pVCpu->hm.s.Event.fPending)
7294 {
7295 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7296 Assert(!pVCpu->hm.s.Event.fPending);
7297 }
7298
7299 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7300 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7301
7302 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7303 and if we're injecting an event we should have a TRPM trap pending. */
7304 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7305#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
7306 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7307#endif
7308
7309 /* Save guest state and restore host state bits. */
7310 int rc = hmR0VmxLeaveSession(pVCpu);
7311 AssertRCReturn(rc, rc);
7312 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7313 /* Thread-context hooks are unregistered at this point!!! */
7314
7315 /* Sync recompiler state. */
7316 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7317 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7318 | CPUM_CHANGED_LDTR
7319 | CPUM_CHANGED_GDTR
7320 | CPUM_CHANGED_IDTR
7321 | CPUM_CHANGED_TR
7322 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7323 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
7324 && CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
7325 {
7326 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7327 }
7328
7329 Assert(!pVCpu->hm.s.fClearTrapFlag);
7330
7331 /* Update the exit-to-ring 3 reason. */
7332 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
7333
7334 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7335 if (rcExit != VINF_EM_RAW_INTERRUPT)
7336 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7337
7338 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7339
7340 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7341 VMMRZCallRing3RemoveNotification(pVCpu);
7342 VMMRZCallRing3Enable(pVCpu);
7343
7344 return rc;
7345}
7346
7347
7348/**
7349 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7350 * longjump to ring-3 and possibly get preempted.
7351 *
7352 * @returns VBox status code.
7353 * @param pVCpu The cross context virtual CPU structure.
7354 * @param enmOperation The operation causing the ring-3 longjump.
7355 * @param pvUser User argument, currently unused, NULL.
7356 */
7357static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7358{
7359 RT_NOREF(pvUser);
7360 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7361 {
7362 /*
7363 * !!! IMPORTANT !!!
7364 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7365 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7366 */
7367 VMMRZCallRing3RemoveNotification(pVCpu);
7368 VMMRZCallRing3Disable(pVCpu);
7369 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7370 RTThreadPreemptDisable(&PreemptState);
7371
7372 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
7373 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7374 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7375
7376#if HC_ARCH_BITS == 64
7377 /* Restore host-state bits that VT-x only restores partially. */
7378 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7379 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7380 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7381 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7382#endif
7383
7384 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7385 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7386 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7387
7388 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7389 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7390 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7391 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7392 {
7393 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7394 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7395 }
7396
7397 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7398 VMMR0ThreadCtxHookDisable(pVCpu);
7399 HMR0LeaveCpu(pVCpu);
7400 RTThreadPreemptRestore(&PreemptState);
7401 return VINF_SUCCESS;
7402 }
7403
7404 Assert(pVCpu);
7405 Assert(pvUser);
7406 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7407 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7408
7409 VMMRZCallRing3Disable(pVCpu);
7410 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7411
7412 Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7413
7414 int rc = hmR0VmxLongJmpToRing3(pVCpu);
7415 AssertRCReturn(rc, rc);
7416
7417 VMMRZCallRing3Enable(pVCpu);
7418 return VINF_SUCCESS;
7419}
7420
7421
7422/**
7423 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7424 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7425 *
7426 * @param pVCpu The cross context virtual CPU structure.
7427 */
7428DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7429{
7430 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7431 {
7432 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7433 {
7434 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_PROC_CTLS_INT_WINDOW_EXIT;
7435 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7436 AssertRC(rc);
7437 Log4Func(("Setup interrupt-window exiting\n"));
7438 }
7439 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7440}
7441
7442
7443/**
7444 * Clears the interrupt-window exiting control in the VMCS.
7445 *
7446 * @param pVCpu The cross context virtual CPU structure.
7447 */
7448DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7449{
7450 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT);
7451 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
7452 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7453 AssertRC(rc);
7454 Log4Func(("Cleared interrupt-window exiting\n"));
7455}
7456
7457
7458/**
7459 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7460 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7461 *
7462 * @param pVCpu The cross context virtual CPU structure.
7463 */
7464DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7465{
7466 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7467 {
7468 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7469 {
7470 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7471 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7472 AssertRC(rc);
7473 Log4Func(("Setup NMI-window exiting\n"));
7474 }
7475 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7476}
7477
7478
7479/**
7480 * Clears the NMI-window exiting control in the VMCS.
7481 *
7482 * @param pVCpu The cross context virtual CPU structure.
7483 */
7484DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7485{
7486 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT);
7487 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7488 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7489 AssertRC(rc);
7490 Log4Func(("Cleared NMI-window exiting\n"));
7491}
7492
7493
7494/**
7495 * Evaluates the event to be delivered to the guest and sets it as the pending
7496 * event.
7497 *
7498 * @returns The VT-x guest-interruptibility state.
7499 * @param pVCpu The cross context virtual CPU structure.
7500 */
7501static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu)
7502{
7503 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7504 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7505 uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu);
7506 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7507 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7508 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7509
7510 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7511 Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7512 Assert(!fBlockSti || pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7513 Assert(!TRPMHasTrap(pVCpu));
7514
7515 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7516 APICUpdatePendingInterrupts(pVCpu);
7517
7518 /*
7519 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7520 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7521 */
7522 /** @todo SMI. SMIs take priority over NMIs. */
7523 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7524 {
7525 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7526 if ( !pVCpu->hm.s.Event.fPending
7527 && !fBlockNmi
7528 && !fBlockSti
7529 && !fBlockMovSS)
7530 {
7531 Log4Func(("Pending NMI\n"));
7532 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INT_INFO_VALID;
7533 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7534
7535 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7536 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7537 }
7538 else
7539 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7540 }
7541 /*
7542 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7543 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7544 */
7545 else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)
7546 && !pVCpu->hm.s.fSingleInstruction)
7547 {
7548 Assert(!DBGFIsStepping(pVCpu));
7549 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7550 AssertRCReturn(rc, 0);
7551 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7552 if ( !pVCpu->hm.s.Event.fPending
7553 && !fBlockInt
7554 && !fBlockSti
7555 && !fBlockMovSS)
7556 {
7557 uint8_t u8Interrupt;
7558 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7559 if (RT_SUCCESS(rc))
7560 {
7561 Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7562 uint32_t u32IntInfo = u8Interrupt
7563 | VMX_EXIT_INT_INFO_VALID
7564 | (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7565
7566 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7567 }
7568 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7569 {
7570 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
7571 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7572 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7573
7574 /*
7575 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7576 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7577 * need to re-set this force-flag here.
7578 */
7579 }
7580 else
7581 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7582 }
7583 else
7584 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7585 }
7586
7587 return fIntrState;
7588}
7589
7590
7591/**
7592 * Sets a pending-debug exception to be delivered to the guest if the guest is
7593 * single-stepping in the VMCS.
7594 *
7595 * @returns VBox status code.
7596 * @param pVCpu The cross context virtual CPU structure.
7597 */
7598DECLINLINE(int) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7599{
7600 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7601 RT_NOREF(pVCpu);
7602 return VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
7603}
7604
7605
7606/**
7607 * Injects any pending events into the guest if the guest is in a state to
7608 * receive them.
7609 *
7610 * @returns Strict VBox status code (i.e. informational status codes too).
7611 * @param pVCpu The cross context virtual CPU structure.
7612 * @param fIntrState The VT-x guest-interruptibility state.
7613 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7614 * return VINF_EM_DBG_STEPPED if the event was
7615 * dispatched directly.
7616 */
7617static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, uint32_t fIntrState, bool fStepping)
7618{
7619 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7620 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7621
7622 bool fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7623 bool fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7624
7625 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7626 Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7627 Assert(!fBlockSti || pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7628 Assert(!TRPMHasTrap(pVCpu));
7629
7630 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7631 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7632 if (pVCpu->hm.s.Event.fPending)
7633 {
7634 /*
7635 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7636 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7637 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7638 *
7639 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7640 */
7641 uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7642#ifdef VBOX_STRICT
7643 if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
7644 {
7645 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7646 Assert(!fBlockInt);
7647 Assert(!fBlockSti);
7648 Assert(!fBlockMovSS);
7649 }
7650 else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
7651 {
7652 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7653 Assert(!fBlockSti);
7654 Assert(!fBlockMovSS);
7655 Assert(!fBlockNmi);
7656 }
7657#endif
7658 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7659 uIntType));
7660 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7661 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7662 &fIntrState);
7663 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7664
7665 /* Update the interruptibility-state as it could have been changed by
7666 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7667 fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7668 fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7669
7670 if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
7671 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7672 else
7673 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7674 }
7675
7676 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7677 if ( fBlockSti
7678 || fBlockMovSS)
7679 {
7680 if (!pVCpu->hm.s.fSingleInstruction)
7681 {
7682 /*
7683 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7684 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7685 * See Intel spec. 27.3.4 "Saving Non-Register State".
7686 */
7687 Assert(!DBGFIsStepping(pVCpu));
7688 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7689 AssertRCReturn(rc, rc);
7690 if (pCtx->eflags.Bits.u1TF)
7691 {
7692 int rc2 = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7693 AssertRCReturn(rc2, rc2);
7694 }
7695 }
7696 else if (pCtx->eflags.Bits.u1TF)
7697 {
7698 /*
7699 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7700 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7701 */
7702 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
7703 fIntrState = 0;
7704 }
7705 }
7706
7707 /*
7708 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7709 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7710 */
7711 int rc3 = hmR0VmxExportGuestIntrState(pVCpu, fIntrState);
7712 AssertRCReturn(rc3, rc3);
7713
7714 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7715 NOREF(fBlockMovSS); NOREF(fBlockSti);
7716 return rcStrict;
7717}
7718
7719
7720/**
7721 * Injects a double-fault (\#DF) exception into the VM.
7722 *
7723 * @returns Strict VBox status code (i.e. informational status codes too).
7724 * @param pVCpu The cross context virtual CPU structure.
7725 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7726 * and should return VINF_EM_DBG_STEPPED if the event
7727 * is injected directly (register modified by us, not
7728 * by hardware on VM-entry).
7729 * @param pfIntrState Pointer to the current guest interruptibility-state.
7730 * This interruptibility-state will be updated if
7731 * necessary. This cannot not be NULL.
7732 */
7733DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, bool fStepping, uint32_t *pfIntrState)
7734{
7735 uint32_t const u32IntInfo = X86_XCPT_DF | VMX_EXIT_INT_INFO_VALID
7736 | (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT)
7737 | VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7738 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */, fStepping,
7739 pfIntrState);
7740}
7741
7742
7743/**
7744 * Injects a general-protection (\#GP) fault into the VM.
7745 *
7746 * @returns Strict VBox status code (i.e. informational status codes too).
7747 * @param pVCpu The cross context virtual CPU structure.
7748 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7749 * mode, i.e. in real-mode it's not valid).
7750 * @param u32ErrorCode The error code associated with the \#GP.
7751 * @param fStepping Whether we're running in
7752 * hmR0VmxRunGuestCodeStep() and should return
7753 * VINF_EM_DBG_STEPPED if the event is injected
7754 * directly (register modified by us, not by
7755 * hardware on VM-entry).
7756 * @param pfIntrState Pointer to the current guest interruptibility-state.
7757 * This interruptibility-state will be updated if
7758 * necessary. This cannot not be NULL.
7759 */
7760DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, bool fErrorCodeValid, uint32_t u32ErrorCode, bool fStepping,
7761 uint32_t *pfIntrState)
7762{
7763 uint32_t const u32IntInfo = X86_XCPT_GP | VMX_EXIT_INT_INFO_VALID
7764 | (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT)
7765 | (fErrorCodeValid ? VMX_EXIT_INT_INFO_ERROR_CODE_VALID : 0);
7766 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */, fStepping,
7767 pfIntrState);
7768}
7769
7770
7771/**
7772 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7773 * stack.
7774 *
7775 * @returns Strict VBox status code (i.e. informational status codes too).
7776 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7777 * @param pVCpu The cross context virtual CPU structure.
7778 * @param uValue The value to push to the guest stack.
7779 */
7780static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
7781{
7782 /*
7783 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7784 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7785 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7786 */
7787 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7788 if (pCtx->sp == 1)
7789 return VINF_EM_RESET;
7790 pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7791 int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
7792 AssertRC(rc);
7793 return rc;
7794}
7795
7796
7797/**
7798 * Injects an event into the guest upon VM-entry by updating the relevant fields
7799 * in the VM-entry area in the VMCS.
7800 *
7801 * @returns Strict VBox status code (i.e. informational status codes too).
7802 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7803 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7804 *
7805 * @param pVCpu The cross context virtual CPU structure.
7806 * @param u64IntInfo The VM-entry interruption-information field.
7807 * @param cbInstr The VM-entry instruction length in bytes (for
7808 * software interrupts, exceptions and privileged
7809 * software exceptions).
7810 * @param u32ErrCode The VM-entry exception error code.
7811 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7812 * @param pfIntrState Pointer to the current guest interruptibility-state.
7813 * This interruptibility-state will be updated if
7814 * necessary. This cannot not be NULL.
7815 * @param fStepping Whether we're running in
7816 * hmR0VmxRunGuestCodeStep() and should return
7817 * VINF_EM_DBG_STEPPED if the event is injected
7818 * directly (register modified by us, not by
7819 * hardware on VM-entry).
7820 */
7821static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7822 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7823{
7824 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7825 AssertMsg(!RT_HI_U32(u64IntInfo), ("%#RX64\n", u64IntInfo));
7826 Assert(pfIntrState);
7827
7828 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7829 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7830 uint32_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
7831 uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
7832
7833#ifdef VBOX_STRICT
7834 /*
7835 * Validate the error-code-valid bit for hardware exceptions.
7836 * No error codes for exceptions in real-mode.
7837 *
7838 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7839 */
7840 if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
7841 && !CPUMIsGuestInRealModeEx(pCtx))
7842 {
7843 switch (uVector)
7844 {
7845 case X86_XCPT_PF:
7846 case X86_XCPT_DF:
7847 case X86_XCPT_TS:
7848 case X86_XCPT_NP:
7849 case X86_XCPT_SS:
7850 case X86_XCPT_GP:
7851 case X86_XCPT_AC:
7852 AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
7853 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7854 RT_FALL_THRU();
7855 default:
7856 break;
7857 }
7858 }
7859#endif
7860
7861 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7862 Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
7863 || !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
7864
7865 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7866
7867 /*
7868 * Hardware interrupts & exceptions cannot be delivered through the software interrupt
7869 * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7870 * interrupt handler in the (real-mode) guest.
7871 *
7872 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7873 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7874 */
7875 if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
7876 {
7877 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7878 {
7879 /*
7880 * For unrestricted execution enabled CPUs running real-mode guests, we must not
7881 * set the deliver-error-code bit.
7882 *
7883 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7884 */
7885 u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
7886 }
7887 else
7888 {
7889 PVM pVM = pVCpu->CTX_SUFF(pVM);
7890 Assert(PDMVmmDevHeapIsEnabled(pVM));
7891 Assert(pVM->hm.s.vmx.pRealModeTSS);
7892
7893 /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7894 int rc2 = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_TABLE_MASK | CPUMCTX_EXTRN_RIP
7895 | CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_RFLAGS);
7896 AssertRCReturn(rc2, rc2);
7897
7898 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7899 size_t const cbIdtEntry = sizeof(X86IDTR16);
7900 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
7901 {
7902 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7903 if (uVector == X86_XCPT_DF)
7904 return VINF_EM_RESET;
7905
7906 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7907 if (uVector == X86_XCPT_GP)
7908 return hmR0VmxInjectXcptDF(pVCpu, fStepping, pfIntrState);
7909
7910 /*
7911 * If we're injecting an event with no valid IDT entry, inject a #GP.
7912 * No error codes for exceptions in real-mode.
7913 *
7914 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7915 */
7916 return hmR0VmxInjectXcptGP(pVCpu, false /* fErrCodeValid */, 0 /* u32ErrCode */, fStepping, pfIntrState);
7917 }
7918
7919 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7920 uint16_t uGuestIp = pCtx->ip;
7921 if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
7922 {
7923 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7924 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7925 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7926 }
7927 else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
7928 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7929
7930 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7931 X86IDTR16 IdtEntry;
7932 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
7933 rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7934 AssertRCReturn(rc2, rc2);
7935
7936 /* Construct the stack frame for the interrupt/exception handler. */
7937 VBOXSTRICTRC rcStrict;
7938 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
7939 if (rcStrict == VINF_SUCCESS)
7940 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
7941 if (rcStrict == VINF_SUCCESS)
7942 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
7943
7944 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7945 if (rcStrict == VINF_SUCCESS)
7946 {
7947 pCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7948 pCtx->rip = IdtEntry.offSel;
7949 pCtx->cs.Sel = IdtEntry.uSel;
7950 pCtx->cs.ValidSel = IdtEntry.uSel;
7951 pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7952 if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
7953 && uVector == X86_XCPT_PF)
7954 pCtx->cr2 = GCPtrFaultAddress;
7955
7956 /* If any other guest-state bits are changed here, make sure to update
7957 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7958 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS | HM_CHANGED_GUEST_CR2
7959 | HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
7960 | HM_CHANGED_GUEST_RSP);
7961
7962 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7963 if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
7964 {
7965 Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
7966 && uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
7967 Log4Func(("Clearing inhibition due to STI\n"));
7968 *pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
7969 }
7970 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7971 u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
7972
7973 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7974 it, if we are returning to ring-3 before executing guest code. */
7975 pVCpu->hm.s.Event.fPending = false;
7976
7977 /* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
7978 if (fStepping)
7979 rcStrict = VINF_EM_DBG_STEPPED;
7980 }
7981 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7982 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7983 return rcStrict;
7984 }
7985 }
7986
7987 /* Validate. */
7988 Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7989 Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
7990
7991 /* Inject. */
7992 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7993 if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
7994 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7995 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7996 AssertRCReturn(rc, rc);
7997
7998 /* Update CR2. */
7999 if ( VMX_ENTRY_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8000 && uVector == X86_XCPT_PF)
8001 pCtx->cr2 = GCPtrFaultAddress;
8002
8003 Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8004
8005 return VINF_SUCCESS;
8006}
8007
8008
8009/**
8010 * Clears the interrupt-window exiting control in the VMCS and if necessary
8011 * clears the current event in the VMCS as well.
8012 *
8013 * @returns VBox status code.
8014 * @param pVCpu The cross context virtual CPU structure.
8015 *
8016 * @remarks Use this function only to clear events that have not yet been
8017 * delivered to the guest but are injected in the VMCS!
8018 * @remarks No-long-jump zone!!!
8019 */
8020static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8021{
8022 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8023 {
8024 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8025 Log4Func(("Cleared interrupt window\n"));
8026 }
8027
8028 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8029 {
8030 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8031 Log4Func(("Cleared NMI window\n"));
8032 }
8033}
8034
8035
8036/**
8037 * Enters the VT-x session.
8038 *
8039 * @returns VBox status code.
8040 * @param pVCpu The cross context virtual CPU structure.
8041 * @param pHostCpu Pointer to the global CPU info struct.
8042 */
8043VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu, PHMGLOBALCPUINFO pHostCpu)
8044{
8045 AssertPtr(pVCpu);
8046 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8047 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8048 RT_NOREF(pHostCpu);
8049
8050 LogFlowFunc(("pVCpu=%p\n", pVCpu));
8051 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8052 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8053
8054#ifdef VBOX_STRICT
8055 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8056 RTCCUINTREG uHostCR4 = ASMGetCR4();
8057 if (!(uHostCR4 & X86_CR4_VMXE))
8058 {
8059 LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8060 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8061 }
8062#endif
8063
8064 /*
8065 * Load the VCPU's VMCS as the current (and active) one.
8066 */
8067 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8068 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8069 if (RT_FAILURE(rc))
8070 return rc;
8071
8072 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8073 pVCpu->hm.s.fLeaveDone = false;
8074 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8075
8076 return VINF_SUCCESS;
8077}
8078
8079
8080/**
8081 * The thread-context callback (only on platforms which support it).
8082 *
8083 * @param enmEvent The thread-context event.
8084 * @param pVCpu The cross context virtual CPU structure.
8085 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8086 * @thread EMT(pVCpu)
8087 */
8088VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8089{
8090 NOREF(fGlobalInit);
8091
8092 switch (enmEvent)
8093 {
8094 case RTTHREADCTXEVENT_OUT:
8095 {
8096 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8097 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8098 VMCPU_ASSERT_EMT(pVCpu);
8099
8100 /* No longjmps (logger flushes, locks) in this fragile context. */
8101 VMMRZCallRing3Disable(pVCpu);
8102 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8103
8104 /*
8105 * Restore host-state (FPU, debug etc.)
8106 */
8107 if (!pVCpu->hm.s.fLeaveDone)
8108 {
8109 /*
8110 * Do -not- import the guest-state here as we might already be in the middle of importing
8111 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
8112 */
8113 hmR0VmxLeave(pVCpu, false /* fImportState */);
8114 pVCpu->hm.s.fLeaveDone = true;
8115 }
8116
8117 /* Leave HM context, takes care of local init (term). */
8118 int rc = HMR0LeaveCpu(pVCpu);
8119 AssertRC(rc); NOREF(rc);
8120
8121 /* Restore longjmp state. */
8122 VMMRZCallRing3Enable(pVCpu);
8123 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8124 break;
8125 }
8126
8127 case RTTHREADCTXEVENT_IN:
8128 {
8129 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8130 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8131 VMCPU_ASSERT_EMT(pVCpu);
8132
8133 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8134 VMMRZCallRing3Disable(pVCpu);
8135 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8136
8137 /* Initialize the bare minimum state required for HM. This takes care of
8138 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8139 int rc = hmR0EnterCpu(pVCpu);
8140 AssertRC(rc);
8141 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8142 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8143
8144 /* Load the active VMCS as the current one. */
8145 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8146 {
8147 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8148 AssertRC(rc); NOREF(rc);
8149 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8150 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8151 }
8152 pVCpu->hm.s.fLeaveDone = false;
8153
8154 /* Restore longjmp state. */
8155 VMMRZCallRing3Enable(pVCpu);
8156 break;
8157 }
8158
8159 default:
8160 break;
8161 }
8162}
8163
8164
8165/**
8166 * Exports the host state into the VMCS host-state area.
8167 * Sets up the VM-exit MSR-load area.
8168 *
8169 * The CPU state will be loaded from these fields on every successful VM-exit.
8170 *
8171 * @returns VBox status code.
8172 * @param pVCpu The cross context virtual CPU structure.
8173 *
8174 * @remarks No-long-jump zone!!!
8175 */
8176static int hmR0VmxExportHostState(PVMCPU pVCpu)
8177{
8178 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8179
8180 int rc = VINF_SUCCESS;
8181 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8182 {
8183 rc = hmR0VmxExportHostControlRegs();
8184 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8185
8186 rc = hmR0VmxExportHostSegmentRegs(pVCpu);
8187 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8188
8189 rc = hmR0VmxExportHostMsrs(pVCpu);
8190 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8191
8192 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
8193 }
8194 return rc;
8195}
8196
8197
8198/**
8199 * Saves the host state in the VMCS host-state.
8200 *
8201 * @returns VBox status code.
8202 * @param pVCpu The cross context virtual CPU structure.
8203 *
8204 * @remarks No-long-jump zone!!!
8205 */
8206VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
8207{
8208 AssertPtr(pVCpu);
8209 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8210
8211 /*
8212 * Export the host state here while entering HM context.
8213 * When thread-context hooks are used, we might get preempted and have to re-save the host
8214 * state but most of the time we won't be, so do it here before we disable interrupts.
8215 */
8216 return hmR0VmxExportHostState(pVCpu);
8217}
8218
8219
8220/**
8221 * Exports the guest state into the VMCS guest-state area.
8222 *
8223 * The will typically be done before VM-entry when the guest-CPU state and the
8224 * VMCS state may potentially be out of sync.
8225 *
8226 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8227 * VM-entry controls.
8228 * Sets up the appropriate VMX non-root function to execute guest code based on
8229 * the guest CPU mode.
8230 *
8231 * @returns VBox strict status code.
8232 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8233 * without unrestricted guest access and the VMMDev is not presently
8234 * mapped (e.g. EFI32).
8235 *
8236 * @param pVCpu The cross context virtual CPU structure.
8237 *
8238 * @remarks No-long-jump zone!!!
8239 */
8240static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu)
8241{
8242 AssertPtr(pVCpu);
8243 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8244
8245 LogFlowFunc(("pVCpu=%p\n", pVCpu));
8246
8247 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
8248
8249 /* Determine real-on-v86 mode. */
8250 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8251 if ( !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
8252 && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
8253 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8254
8255 /*
8256 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8257 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8258 */
8259 int rc = hmR0VmxSelectVMRunHandler(pVCpu);
8260 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8261
8262 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8263 rc = hmR0VmxExportGuestEntryCtls(pVCpu);
8264 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8265
8266 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8267 rc = hmR0VmxExportGuestExitCtls(pVCpu);
8268 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8269
8270 rc = hmR0VmxExportGuestCR0(pVCpu);
8271 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8272
8273 VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu);
8274 if (rcStrict == VINF_SUCCESS)
8275 { /* likely */ }
8276 else
8277 {
8278 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8279 return rcStrict;
8280 }
8281
8282 rc = hmR0VmxExportGuestSegmentRegs(pVCpu);
8283 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8284
8285 /* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8286 may alter controls if we determine we don't have to swap EFER after all. */
8287 rc = hmR0VmxExportGuestMsrs(pVCpu);
8288 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8289
8290 rc = hmR0VmxExportGuestApicTpr(pVCpu);
8291 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8292
8293 rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8294 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8295
8296 /* Exporting RFLAGS here is fine, even though RFLAGS.TF might depend on guest debug state which is
8297 not exported here. It is re-evaluated and updated if necessary in hmR0VmxExportSharedState(). */
8298 rc = hmR0VmxExportGuestRip(pVCpu);
8299 rc |= hmR0VmxExportGuestRsp(pVCpu);
8300 rc |= hmR0VmxExportGuestRflags(pVCpu);
8301 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8302
8303 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8304 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8305 | HM_CHANGED_GUEST_CR2
8306 | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8307 | HM_CHANGED_GUEST_X87
8308 | HM_CHANGED_GUEST_SSE_AVX
8309 | HM_CHANGED_GUEST_OTHER_XSAVE
8310 | HM_CHANGED_GUEST_XCRx
8311 | HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8312 | HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8313 | HM_CHANGED_GUEST_TSC_AUX
8314 | HM_CHANGED_GUEST_OTHER_MSRS
8315 | HM_CHANGED_GUEST_HWVIRT
8316 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8317
8318 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8319 return rc;
8320}
8321
8322
8323/**
8324 * Exports the state shared between the host and guest into the VMCS.
8325 *
8326 * @param pVCpu The cross context virtual CPU structure.
8327 *
8328 * @remarks No-long-jump zone!!!
8329 */
8330static void hmR0VmxExportSharedState(PVMCPU pVCpu)
8331{
8332 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8333 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8334
8335 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8336 {
8337 int rc = hmR0VmxExportSharedDebugState(pVCpu);
8338 AssertRC(rc);
8339 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8340
8341 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8342 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8343 {
8344 rc = hmR0VmxExportGuestRflags(pVCpu);
8345 AssertRC(rc);
8346 }
8347 }
8348
8349 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8350 {
8351 hmR0VmxLazyLoadGuestMsrs(pVCpu);
8352 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8353 }
8354
8355 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8356 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8357}
8358
8359
8360/**
8361 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8362 *
8363 * @returns Strict VBox status code (i.e. informational status codes too).
8364 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8365 * without unrestricted guest access and the VMMDev is not presently
8366 * mapped (e.g. EFI32).
8367 *
8368 * @param pVCpu The cross context virtual CPU structure.
8369 *
8370 * @remarks No-long-jump zone!!!
8371 */
8372static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu)
8373{
8374 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8375 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8376 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8377
8378#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8379 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8380#endif
8381
8382 /*
8383 * For many exits it's only RIP that changes and hence try to export it first
8384 * without going through a lot of change flag checks.
8385 */
8386 VBOXSTRICTRC rcStrict;
8387 uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8388 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8389 if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8390 {
8391 rcStrict = hmR0VmxExportGuestRip(pVCpu);
8392 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8393 { /* likely */}
8394 else
8395 AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8396 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8397 }
8398 else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8399 {
8400 rcStrict = hmR0VmxExportGuestState(pVCpu);
8401 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8402 { /* likely */}
8403 else
8404 {
8405 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8406 VBOXSTRICTRC_VAL(rcStrict)));
8407 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8408 return rcStrict;
8409 }
8410 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8411 }
8412 else
8413 rcStrict = VINF_SUCCESS;
8414
8415#ifdef VBOX_STRICT
8416 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8417 fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8418 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8419 AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8420 ("fCtxChanged=%#RX64\n", fCtxChanged));
8421#endif
8422 return rcStrict;
8423}
8424
8425
8426/**
8427 * Does the preparations before executing guest code in VT-x.
8428 *
8429 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8430 * recompiler/IEM. We must be cautious what we do here regarding committing
8431 * guest-state information into the VMCS assuming we assuredly execute the
8432 * guest in VT-x mode.
8433 *
8434 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8435 * the common-state (TRPM/forceflags), we must undo those changes so that the
8436 * recompiler/IEM can (and should) use them when it resumes guest execution.
8437 * Otherwise such operations must be done when we can no longer exit to ring-3.
8438 *
8439 * @returns Strict VBox status code (i.e. informational status codes too).
8440 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8441 * have been disabled.
8442 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8443 * double-fault into the guest.
8444 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8445 * dispatched directly.
8446 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8447 *
8448 * @param pVCpu The cross context virtual CPU structure.
8449 * @param pVmxTransient Pointer to the VMX transient structure.
8450 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8451 * us ignore some of the reasons for returning to
8452 * ring-3, and return VINF_EM_DBG_STEPPED if event
8453 * dispatching took place.
8454 */
8455static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
8456{
8457 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8458
8459#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_ONLY_IN_IEM
8460 Log2(("hmR0SvmPreRunGuest: Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
8461 return VINF_EM_RESCHEDULE_REM;
8462#endif
8463
8464#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8465 PGMRZDynMapFlushAutoSet(pVCpu);
8466#endif
8467
8468 /* Check force flag actions that might require us to go back to ring-3. */
8469 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
8470 if (rcStrict == VINF_SUCCESS)
8471 { /* FFs doesn't get set all the time. */ }
8472 else
8473 return rcStrict;
8474
8475 /*
8476 * Setup the virtualized-APIC accesses.
8477 *
8478 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8479 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8480 *
8481 * This is the reason we do it here and not in hmR0VmxExportGuestState().
8482 */
8483 PVM pVM = pVCpu->CTX_SUFF(pVM);
8484 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8485 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
8486 && PDMHasApic(pVM))
8487 {
8488 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8489 Assert(u64MsrApicBase);
8490 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8491
8492 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8493
8494 /* Unalias any existing mapping. */
8495 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8496 AssertRCReturn(rc, rc);
8497
8498 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8499 Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8500 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8501 AssertRCReturn(rc, rc);
8502
8503 /* Update the per-VCPU cache of the APIC base MSR. */
8504 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8505 }
8506
8507 if (TRPMHasTrap(pVCpu))
8508 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8509 uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu);
8510
8511 /*
8512 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
8513 * needs to be done with longjmps or interrupts + preemption enabled. Event injection might
8514 * also result in triple-faulting the VM.
8515 */
8516 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, fIntrState, fStepping);
8517 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8518 { /* likely */ }
8519 else
8520 {
8521 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8522 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8523 return rcStrict;
8524 }
8525
8526 /*
8527 * A longjump might result in importing CR3 even for VM-exits that don't necessarily
8528 * import CR3 themselves. We will need to update them here, as even as late as the above
8529 * hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
8530 * the below force flags to be set.
8531 */
8532 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8533 {
8534 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
8535 int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8536 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
8537 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
8538 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8539 }
8540 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8541 {
8542 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8543 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8544 }
8545
8546 /*
8547 * No longjmps to ring-3 from this point on!!!
8548 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8549 * This also disables flushing of the R0-logger instance (if any).
8550 */
8551 VMMRZCallRing3Disable(pVCpu);
8552
8553 /*
8554 * Export the guest state bits.
8555 *
8556 * We cannot perform longjmps while loading the guest state because we do not preserve the
8557 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8558 * CPU migration.
8559 *
8560 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8561 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8562 * Hence, loading of the guest state needs to be done -after- injection of events.
8563 */
8564 rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu);
8565 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8566 { /* likely */ }
8567 else
8568 {
8569 VMMRZCallRing3Enable(pVCpu);
8570 return rcStrict;
8571 }
8572
8573 /*
8574 * We disable interrupts so that we don't miss any interrupts that would flag preemption
8575 * (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
8576 * preemption disabled for a while. Since this is purly to aid the
8577 * RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
8578 * disable interrupt on NT.
8579 *
8580 * We need to check for force-flags that could've possible been altered since we last
8581 * checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
8582 * see @bugref{6398}).
8583 *
8584 * We also check a couple of other force-flags as a last opportunity to get the EMT back
8585 * to ring-3 before executing guest code.
8586 */
8587 pVmxTransient->fEFlags = ASMIntDisableFlags();
8588
8589 if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8590 && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8591 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8592 && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8593 {
8594 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8595 {
8596 pVCpu->hm.s.Event.fPending = false;
8597
8598 /*
8599 * We've injected any pending events. This is really the point of no return (to ring-3).
8600 *
8601 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8602 * returns from this function, so don't enable them here.
8603 */
8604 return VINF_SUCCESS;
8605 }
8606
8607 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
8608 rcStrict = VINF_EM_RAW_INTERRUPT;
8609 }
8610 else
8611 {
8612 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8613 rcStrict = VINF_EM_RAW_TO_R3;
8614 }
8615
8616 ASMSetFlags(pVmxTransient->fEFlags);
8617 VMMRZCallRing3Enable(pVCpu);
8618
8619 return rcStrict;
8620}
8621
8622
8623/**
8624 * Prepares to run guest code in VT-x and we've committed to doing so. This
8625 * means there is no backing out to ring-3 or anywhere else at this
8626 * point.
8627 *
8628 * @param pVCpu The cross context virtual CPU structure.
8629 * @param pVmxTransient Pointer to the VMX transient structure.
8630 *
8631 * @remarks Called with preemption disabled.
8632 * @remarks No-long-jump zone!!!
8633 */
8634static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
8635{
8636 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8637 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8638 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8639
8640 /*
8641 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8642 */
8643 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8644 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8645
8646 PVM pVM = pVCpu->CTX_SUFF(pVM);
8647 if (!CPUMIsGuestFPUStateActive(pVCpu))
8648 {
8649 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8650 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8651 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
8652 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8653 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8654 }
8655
8656 /*
8657 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8658 */
8659 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8660 && pVCpu->hm.s.vmx.cMsrs > 0)
8661 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8662
8663 /*
8664 * Re-save the host state bits as we may've been preempted (only happens when
8665 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8666 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8667 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8668 * See @bugref{8432}.
8669 */
8670 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8671 {
8672 int rc = hmR0VmxExportHostState(pVCpu);
8673 AssertRC(rc);
8674 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8675 }
8676 Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8677
8678 /*
8679 * Export the state shared between host and guest (FPU, debug, lazy MSRs).
8680 */
8681 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8682 hmR0VmxExportSharedState(pVCpu);
8683 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8684
8685 /* Store status of the shared guest-host state at the time of VM-entry. */
8686#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8687 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
8688 {
8689 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8690 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8691 }
8692 else
8693#endif
8694 {
8695 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8696 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8697 }
8698
8699 /*
8700 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8701 */
8702 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
8703 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8704
8705 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8706 RTCPUID idCurrentCpu = pCpu->idCpu;
8707 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8708 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8709 {
8710 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8711 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8712 }
8713
8714 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8715 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8716 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8717 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8718
8719 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8720
8721 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8722 to start executing. */
8723
8724 /*
8725 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8726 */
8727 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
8728 {
8729 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
8730 {
8731 bool fMsrUpdated;
8732 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8733 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8734 &fMsrUpdated);
8735 AssertRC(rc2);
8736 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8737 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8738 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8739 }
8740 else
8741 {
8742 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8743 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8744 }
8745 }
8746
8747 if (pVM->cpum.ro.GuestFeatures.fIbrs)
8748 {
8749 bool fMsrUpdated;
8750 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8751 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8752 &fMsrUpdated);
8753 AssertRC(rc2);
8754 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8755 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8756 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8757 }
8758
8759#ifdef VBOX_STRICT
8760 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8761 hmR0VmxCheckHostEferMsr(pVCpu);
8762 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8763#endif
8764#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8765 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
8766 {
8767 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
8768 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8769 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8770 }
8771#endif
8772}
8773
8774
8775/**
8776 * Performs some essential restoration of state after running guest code in
8777 * VT-x.
8778 *
8779 * @param pVCpu The cross context virtual CPU structure.
8780 * @param pVmxTransient Pointer to the VMX transient structure.
8781 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8782 *
8783 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8784 *
8785 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8786 * unconditionally when it is safe to do so.
8787 */
8788static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8789{
8790 uint64_t const uHostTsc = ASMReadTSC();
8791 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8792
8793 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8794 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8795 pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8796 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8797 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8798 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8799
8800 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
8801 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TscOffset);
8802
8803 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
8804 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8805 Assert(!ASMIntAreEnabled());
8806 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8807
8808#if HC_ARCH_BITS == 64
8809 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8810#endif
8811#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8812 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8813 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8814 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8815#else
8816 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8817#endif
8818#ifdef VBOX_STRICT
8819 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8820#endif
8821 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8822
8823 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8824 uint32_t uExitReason;
8825 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8826 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8827 AssertRC(rc);
8828 pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
8829 pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
8830
8831 if (rcVMRun == VINF_SUCCESS)
8832 {
8833 /*
8834 * Update the VM-exit history array here even if the VM-entry failed due to:
8835 * - Invalid guest state.
8836 * - MSR loading.
8837 * - Machine-check event.
8838 *
8839 * In any of the above cases we will still have a "valid" VM-exit reason
8840 * despite @a fVMEntryFailed being false.
8841 *
8842 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8843 *
8844 * Note! We don't have CS or RIP at this point. Will probably address that later
8845 * by amending the history entry added here.
8846 */
8847 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8848 UINT64_MAX, uHostTsc);
8849
8850 if (!pVmxTransient->fVMEntryFailed)
8851 {
8852 VMMRZCallRing3Enable(pVCpu);
8853
8854 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8855 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8856
8857#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8858 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8859 AssertRC(rc);
8860#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8861 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8862 AssertRC(rc);
8863#else
8864 /*
8865 * Import the guest-interruptibility state always as we need it while evaluating
8866 * injecting events on re-entry.
8867 *
8868 * We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8869 * checking for real-mode while exporting the state because all bits that cause
8870 * mode changes wrt CR0 are intercepted.
8871 */
8872 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8873 AssertRC(rc);
8874#endif
8875
8876 /*
8877 * Sync the TPR shadow with our APIC state.
8878 */
8879 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
8880 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8881 {
8882 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8883 AssertRC(rc);
8884 ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8885 }
8886
8887 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8888 return;
8889 }
8890 }
8891 else
8892 Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8893
8894 VMMRZCallRing3Enable(pVCpu);
8895}
8896
8897
8898/**
8899 * Runs the guest code using VT-x the normal way.
8900 *
8901 * @returns VBox status code.
8902 * @param pVCpu The cross context virtual CPU structure.
8903 *
8904 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8905 */
8906static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu)
8907{
8908 VMXTRANSIENT VmxTransient;
8909 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8910 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8911 uint32_t cLoops = 0;
8912
8913 for (;; cLoops++)
8914 {
8915 Assert(!HMR0SuspendPending());
8916 HMVMX_ASSERT_CPU_SAFE(pVCpu);
8917
8918 /* Preparatory work for running guest code, this may force us to return
8919 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8920 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8921 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
8922 if (rcStrict != VINF_SUCCESS)
8923 break;
8924
8925 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
8926 int rcRun = hmR0VmxRunGuest(pVCpu);
8927
8928 /* Restore any residual host-state and save any bits shared between host
8929 and guest into the guest-CPU state. Re-enables interrupts! */
8930 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8931
8932 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8933 if (RT_SUCCESS(rcRun))
8934 { /* very likely */ }
8935 else
8936 {
8937 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
8938 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
8939 return rcRun;
8940 }
8941
8942 /* Profile the VM-exit. */
8943 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8944 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8945 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8946 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
8947 HMVMX_START_EXIT_DISPATCH_PROF();
8948
8949 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
8950
8951 /* Handle the VM-exit. */
8952#ifdef HMVMX_USE_FUNCTION_TABLE
8953 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
8954#else
8955 rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient, VmxTransient.uExitReason);
8956#endif
8957 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
8958 if (rcStrict == VINF_SUCCESS)
8959 {
8960 if (cLoops <= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
8961 continue; /* likely */
8962 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8963 rcStrict = VINF_EM_RAW_INTERRUPT;
8964 }
8965 break;
8966 }
8967
8968 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8969 return rcStrict;
8970}
8971
8972
8973
8974/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8975 * probes.
8976 *
8977 * The following few functions and associated structure contains the bloat
8978 * necessary for providing detailed debug events and dtrace probes as well as
8979 * reliable host side single stepping. This works on the principle of
8980 * "subclassing" the normal execution loop and workers. We replace the loop
8981 * method completely and override selected helpers to add necessary adjustments
8982 * to their core operation.
8983 *
8984 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8985 * any performance for debug and analysis features.
8986 *
8987 * @{
8988 */
8989
8990/**
8991 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
8992 * the debug run loop.
8993 */
8994typedef struct VMXRUNDBGSTATE
8995{
8996 /** The RIP we started executing at. This is for detecting that we stepped. */
8997 uint64_t uRipStart;
8998 /** The CS we started executing with. */
8999 uint16_t uCsStart;
9000
9001 /** Whether we've actually modified the 1st execution control field. */
9002 bool fModifiedProcCtls : 1;
9003 /** Whether we've actually modified the 2nd execution control field. */
9004 bool fModifiedProcCtls2 : 1;
9005 /** Whether we've actually modified the exception bitmap. */
9006 bool fModifiedXcptBitmap : 1;
9007
9008 /** We desire the modified the CR0 mask to be cleared. */
9009 bool fClearCr0Mask : 1;
9010 /** We desire the modified the CR4 mask to be cleared. */
9011 bool fClearCr4Mask : 1;
9012 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9013 uint32_t fCpe1Extra;
9014 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9015 uint32_t fCpe1Unwanted;
9016 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9017 uint32_t fCpe2Extra;
9018 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9019 uint32_t bmXcptExtra;
9020 /** The sequence number of the Dtrace provider settings the state was
9021 * configured against. */
9022 uint32_t uDtraceSettingsSeqNo;
9023 /** VM-exits to check (one bit per VM-exit). */
9024 uint32_t bmExitsToCheck[3];
9025
9026 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9027 uint32_t fProcCtlsInitial;
9028 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9029 uint32_t fProcCtls2Initial;
9030 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9031 uint32_t bmXcptInitial;
9032} VMXRUNDBGSTATE;
9033AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9034typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9035
9036
9037/**
9038 * Initializes the VMXRUNDBGSTATE structure.
9039 *
9040 * @param pVCpu The cross context virtual CPU structure of the
9041 * calling EMT.
9042 * @param pDbgState The structure to initialize.
9043 */
9044static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9045{
9046 pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
9047 pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
9048
9049 pDbgState->fModifiedProcCtls = false;
9050 pDbgState->fModifiedProcCtls2 = false;
9051 pDbgState->fModifiedXcptBitmap = false;
9052 pDbgState->fClearCr0Mask = false;
9053 pDbgState->fClearCr4Mask = false;
9054 pDbgState->fCpe1Extra = 0;
9055 pDbgState->fCpe1Unwanted = 0;
9056 pDbgState->fCpe2Extra = 0;
9057 pDbgState->bmXcptExtra = 0;
9058 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9059 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9060 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9061}
9062
9063
9064/**
9065 * Updates the VMSC fields with changes requested by @a pDbgState.
9066 *
9067 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9068 * immediately before executing guest code, i.e. when interrupts are disabled.
9069 * We don't check status codes here as we cannot easily assert or return in the
9070 * latter case.
9071 *
9072 * @param pVCpu The cross context virtual CPU structure.
9073 * @param pDbgState The debug state.
9074 */
9075static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9076{
9077 /*
9078 * Ensure desired flags in VMCS control fields are set.
9079 * (Ignoring write failure here, as we're committed and it's just debug extras.)
9080 *
9081 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9082 * there should be no stale data in pCtx at this point.
9083 */
9084 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9085 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9086 {
9087 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
9088 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9089 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9090 Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9091 pDbgState->fModifiedProcCtls = true;
9092 }
9093
9094 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9095 {
9096 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9097 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9098 Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9099 pDbgState->fModifiedProcCtls2 = true;
9100 }
9101
9102 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9103 {
9104 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9105 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9106 Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9107 pDbgState->fModifiedXcptBitmap = true;
9108 }
9109
9110 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32Cr0Mask != 0)
9111 {
9112 pVCpu->hm.s.vmx.u32Cr0Mask = 0;
9113 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9114 Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9115 }
9116
9117 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32Cr4Mask != 0)
9118 {
9119 pVCpu->hm.s.vmx.u32Cr4Mask = 0;
9120 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9121 Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9122 }
9123}
9124
9125
9126/**
9127 * Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
9128 * re-entry next time around.
9129 *
9130 * @returns Strict VBox status code (i.e. informational status codes too).
9131 * @param pVCpu The cross context virtual CPU structure.
9132 * @param pDbgState The debug state.
9133 * @param rcStrict The return code from executing the guest using single
9134 * stepping.
9135 */
9136static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9137{
9138 /*
9139 * Restore VM-exit control settings as we may not reenter this function the
9140 * next time around.
9141 */
9142 /* We reload the initial value, trigger what we can of recalculations the
9143 next time around. From the looks of things, that's all that's required atm. */
9144 if (pDbgState->fModifiedProcCtls)
9145 {
9146 if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9147 pDbgState->fProcCtlsInitial |= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9148 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9149 AssertRCReturn(rc2, rc2);
9150 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9151 }
9152
9153 /* We're currently the only ones messing with this one, so just restore the
9154 cached value and reload the field. */
9155 if ( pDbgState->fModifiedProcCtls2
9156 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9157 {
9158 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9159 AssertRCReturn(rc2, rc2);
9160 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9161 }
9162
9163 /* If we've modified the exception bitmap, we restore it and trigger
9164 reloading and partial recalculation the next time around. */
9165 if (pDbgState->fModifiedXcptBitmap)
9166 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9167
9168 return rcStrict;
9169}
9170
9171
9172/**
9173 * Configures VM-exit controls for current DBGF and DTrace settings.
9174 *
9175 * This updates @a pDbgState and the VMCS execution control fields to reflect
9176 * the necessary VM-exits demanded by DBGF and DTrace.
9177 *
9178 * @param pVCpu The cross context virtual CPU structure.
9179 * @param pDbgState The debug state.
9180 * @param pVmxTransient Pointer to the VMX transient structure. May update
9181 * fUpdateTscOffsettingAndPreemptTimer.
9182 */
9183static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9184{
9185 /*
9186 * Take down the dtrace serial number so we can spot changes.
9187 */
9188 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9189 ASMCompilerBarrier();
9190
9191 /*
9192 * We'll rebuild most of the middle block of data members (holding the
9193 * current settings) as we go along here, so start by clearing it all.
9194 */
9195 pDbgState->bmXcptExtra = 0;
9196 pDbgState->fCpe1Extra = 0;
9197 pDbgState->fCpe1Unwanted = 0;
9198 pDbgState->fCpe2Extra = 0;
9199 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9200 pDbgState->bmExitsToCheck[i] = 0;
9201
9202 /*
9203 * Software interrupts (INT XXh) - no idea how to trigger these...
9204 */
9205 PVM pVM = pVCpu->CTX_SUFF(pVM);
9206 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9207 || VBOXVMM_INT_SOFTWARE_ENABLED())
9208 {
9209 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9210 }
9211
9212 /*
9213 * INT3 breakpoints - triggered by #BP exceptions.
9214 */
9215 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9216 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9217
9218 /*
9219 * Exception bitmap and XCPT events+probes.
9220 */
9221 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9222 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9223 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9224
9225 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9226 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9227 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9228 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9229 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9230 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9231 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9232 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9233 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9234 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9235 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9236 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9237 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9238 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9239 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9240 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9241 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9242 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9243
9244 if (pDbgState->bmXcptExtra)
9245 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9246
9247 /*
9248 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9249 *
9250 * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
9251 * So, when adding/changing/removing please don't forget to update it.
9252 *
9253 * Some of the macros are picking up local variables to save horizontal space,
9254 * (being able to see it in a table is the lesser evil here).
9255 */
9256#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9257 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9258 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9259#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9260 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9261 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9262 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9263 } else do { } while (0)
9264#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9265 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9266 { \
9267 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9268 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9269 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9270 } else do { } while (0)
9271#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9272 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9273 { \
9274 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9275 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9276 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9277 } else do { } while (0)
9278#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9279 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9280 { \
9281 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9282 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9283 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9284 } else do { } while (0)
9285
9286 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9287 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9288 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9289 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9290 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9291
9292 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9293 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9294 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9295 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9296 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
9297 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9298 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9299 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9300 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
9301 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9302 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
9303 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9304 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
9305 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9306 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9307 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9308 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9309 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9310 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9311 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9312 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9313 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9314 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9315 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9316 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9317 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9318 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9319 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9320 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9321 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9322 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9323 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9324 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9325 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9326 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9327 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9328
9329 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9330 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9331 {
9332 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_APIC_TPR);
9333 AssertRC(rc);
9334
9335#if 0 /** @todo fix me */
9336 pDbgState->fClearCr0Mask = true;
9337 pDbgState->fClearCr4Mask = true;
9338#endif
9339 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9340 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_STORE_EXIT | VMX_PROC_CTLS_CR8_STORE_EXIT;
9341 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9342 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_LOAD_EXIT | VMX_PROC_CTLS_CR8_LOAD_EXIT;
9343 pDbgState->fCpe1Unwanted |= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
9344 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9345 require clearing here and in the loop if we start using it. */
9346 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9347 }
9348 else
9349 {
9350 if (pDbgState->fClearCr0Mask)
9351 {
9352 pDbgState->fClearCr0Mask = false;
9353 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9354 }
9355 if (pDbgState->fClearCr4Mask)
9356 {
9357 pDbgState->fClearCr4Mask = false;
9358 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9359 }
9360 }
9361 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9362 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9363
9364 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9365 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9366 {
9367 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9368 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9369 }
9370 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9371 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9372
9373 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
9374 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9375 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
9376 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9377 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
9378 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9379 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
9380 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9381#if 0 /** @todo too slow, fix handler. */
9382 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
9383#endif
9384 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9385
9386 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9387 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9388 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9389 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9390 {
9391 pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
9392 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
9393 }
9394 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9395 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9396 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9397 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9398
9399 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9400 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9401 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9402 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9403 {
9404 pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
9405 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
9406 }
9407 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
9408 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
9409 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
9410 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
9411
9412 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9413 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9414 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
9415 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9416 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9417 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9418 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
9419 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9420 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9421 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9422 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
9423 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9424 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
9425 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9426 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9427 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9428 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
9429 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9430 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9431 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9432 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9433 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9434
9435#undef IS_EITHER_ENABLED
9436#undef SET_ONLY_XBM_IF_EITHER_EN
9437#undef SET_CPE1_XBM_IF_EITHER_EN
9438#undef SET_CPEU_XBM_IF_EITHER_EN
9439#undef SET_CPE2_XBM_IF_EITHER_EN
9440
9441 /*
9442 * Sanitize the control stuff.
9443 */
9444 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
9445 if (pDbgState->fCpe2Extra)
9446 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
9447 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
9448 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0;
9449 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
9450 {
9451 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9452 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9453 }
9454
9455 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9456 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9457 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9458 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9459}
9460
9461
9462/**
9463 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9464 * appropriate.
9465 *
9466 * The caller has checked the VM-exit against the
9467 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9468 * already, so we don't have to do that either.
9469 *
9470 * @returns Strict VBox status code (i.e. informational status codes too).
9471 * @param pVCpu The cross context virtual CPU structure.
9472 * @param pVmxTransient Pointer to the VMX-transient structure.
9473 * @param uExitReason The VM-exit reason.
9474 *
9475 * @remarks The name of this function is displayed by dtrace, so keep it short
9476 * and to the point. No longer than 33 chars long, please.
9477 */
9478static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9479{
9480 /*
9481 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9482 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9483 *
9484 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9485 * does. Must add/change/remove both places. Same ordering, please.
9486 *
9487 * Added/removed events must also be reflected in the next section
9488 * where we dispatch dtrace events.
9489 */
9490 bool fDtrace1 = false;
9491 bool fDtrace2 = false;
9492 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9493 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9494 uint32_t uEventArg = 0;
9495#define SET_EXIT(a_EventSubName) \
9496 do { \
9497 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9498 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9499 } while (0)
9500#define SET_BOTH(a_EventSubName) \
9501 do { \
9502 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9503 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9504 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9505 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9506 } while (0)
9507 switch (uExitReason)
9508 {
9509 case VMX_EXIT_MTF:
9510 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9511
9512 case VMX_EXIT_XCPT_OR_NMI:
9513 {
9514 uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9515 switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
9516 {
9517 case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
9518 case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
9519 case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
9520 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9521 {
9522 if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
9523 {
9524 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9525 uEventArg = pVmxTransient->uExitIntErrorCode;
9526 }
9527 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9528 switch (enmEvent1)
9529 {
9530 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9531 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9532 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9533 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9534 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9535 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9536 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9537 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9538 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9539 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9540 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9541 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9542 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9543 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9544 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9545 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9546 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9547 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9548 default: break;
9549 }
9550 }
9551 else
9552 AssertFailed();
9553 break;
9554
9555 case VMX_EXIT_INT_INFO_TYPE_SW_INT:
9556 uEventArg = idxVector;
9557 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9558 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9559 break;
9560 }
9561 break;
9562 }
9563
9564 case VMX_EXIT_TRIPLE_FAULT:
9565 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9566 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9567 break;
9568 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9569 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9570 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9571 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9572 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9573
9574 /* Instruction specific VM-exits: */
9575 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9576 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9577 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9578 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9579 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9580 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9581 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9582 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9583 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9584 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9585 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9586 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9587 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9588 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9589 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9590 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9591 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9592 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9593 case VMX_EXIT_MOV_CRX:
9594 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9595 if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9596 SET_BOTH(CRX_READ);
9597 else
9598 SET_BOTH(CRX_WRITE);
9599 uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
9600 break;
9601 case VMX_EXIT_MOV_DRX:
9602 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9603 if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
9604 == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9605 SET_BOTH(DRX_READ);
9606 else
9607 SET_BOTH(DRX_WRITE);
9608 uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
9609 break;
9610 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9611 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9612 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9613 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9614 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9615 case VMX_EXIT_GDTR_IDTR_ACCESS:
9616 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9617 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
9618 {
9619 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9620 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9621 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9622 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9623 }
9624 break;
9625
9626 case VMX_EXIT_LDTR_TR_ACCESS:
9627 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9628 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
9629 {
9630 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9631 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9632 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9633 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9634 }
9635 break;
9636
9637 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9638 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9639 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9640 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9641 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9642 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9643 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9644 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9645 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9646 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9647 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9648
9649 /* Events that aren't relevant at this point. */
9650 case VMX_EXIT_EXT_INT:
9651 case VMX_EXIT_INT_WINDOW:
9652 case VMX_EXIT_NMI_WINDOW:
9653 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9654 case VMX_EXIT_PREEMPT_TIMER:
9655 case VMX_EXIT_IO_INSTR:
9656 break;
9657
9658 /* Errors and unexpected events. */
9659 case VMX_EXIT_INIT_SIGNAL:
9660 case VMX_EXIT_SIPI:
9661 case VMX_EXIT_IO_SMI:
9662 case VMX_EXIT_SMI:
9663 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9664 case VMX_EXIT_ERR_MSR_LOAD:
9665 case VMX_EXIT_ERR_MACHINE_CHECK:
9666 break;
9667
9668 default:
9669 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9670 break;
9671 }
9672#undef SET_BOTH
9673#undef SET_EXIT
9674
9675 /*
9676 * Dtrace tracepoints go first. We do them here at once so we don't
9677 * have to copy the guest state saving and stuff a few dozen times.
9678 * Down side is that we've got to repeat the switch, though this time
9679 * we use enmEvent since the probes are a subset of what DBGF does.
9680 */
9681 if (fDtrace1 || fDtrace2)
9682 {
9683 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9684 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9685 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9686 switch (enmEvent1)
9687 {
9688 /** @todo consider which extra parameters would be helpful for each probe. */
9689 case DBGFEVENT_END: break;
9690 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
9691 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
9692 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
9693 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
9694 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
9695 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
9696 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
9697 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
9698 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
9699 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
9700 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
9701 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
9702 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
9703 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
9704 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
9705 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
9706 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
9707 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
9708 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9709 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9710 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
9711 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
9712 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
9713 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
9714 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
9715 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
9716 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
9717 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9718 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9719 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9720 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9721 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9722 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
9723 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9724 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
9725 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
9726 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
9727 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
9728 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
9729 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
9730 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
9731 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
9732 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
9733 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
9734 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
9735 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
9736 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
9737 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
9738 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
9739 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
9740 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
9741 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
9742 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
9743 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
9744 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
9745 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
9746 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
9747 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
9748 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
9749 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
9750 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
9751 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
9752 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
9753 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
9754 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
9755 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
9756 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9757 }
9758 switch (enmEvent2)
9759 {
9760 /** @todo consider which extra parameters would be helpful for each probe. */
9761 case DBGFEVENT_END: break;
9762 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
9763 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9764 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
9765 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
9766 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
9767 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
9768 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
9769 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
9770 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
9771 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9772 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9773 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9774 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9775 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9776 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
9777 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9778 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
9779 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
9780 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
9781 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
9782 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
9783 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
9784 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
9785 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
9786 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
9787 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
9788 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
9789 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
9790 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
9791 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
9792 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
9793 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
9794 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
9795 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
9796 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
9797 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
9798 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
9799 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
9800 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
9801 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
9802 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
9803 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
9804 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
9805 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
9806 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
9807 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
9808 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
9809 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
9810 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
9811 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
9812 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
9813 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
9814 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9815 }
9816 }
9817
9818 /*
9819 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9820 * the DBGF call will do a full check).
9821 *
9822 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9823 * Note! If we have to events, we prioritize the first, i.e. the instruction
9824 * one, in order to avoid event nesting.
9825 */
9826 PVM pVM = pVCpu->CTX_SUFF(pVM);
9827 if ( enmEvent1 != DBGFEVENT_END
9828 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9829 {
9830 HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
9831 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
9832 if (rcStrict != VINF_SUCCESS)
9833 return rcStrict;
9834 }
9835 else if ( enmEvent2 != DBGFEVENT_END
9836 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9837 {
9838 HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
9839 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
9840 if (rcStrict != VINF_SUCCESS)
9841 return rcStrict;
9842 }
9843
9844 return VINF_SUCCESS;
9845}
9846
9847
9848/**
9849 * Single-stepping VM-exit filtering.
9850 *
9851 * This is preprocessing the VM-exits and deciding whether we've gotten far
9852 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9853 * handling is performed.
9854 *
9855 * @returns Strict VBox status code (i.e. informational status codes too).
9856 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9857 * @param pVmxTransient Pointer to the VMX-transient structure.
9858 * @param pDbgState The debug state.
9859 */
9860DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
9861{
9862 /*
9863 * Expensive (saves context) generic dtrace VM-exit probe.
9864 */
9865 uint32_t const uExitReason = pVmxTransient->uExitReason;
9866 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9867 { /* more likely */ }
9868 else
9869 {
9870 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9871 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9872 AssertRC(rc);
9873 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
9874 }
9875
9876 /*
9877 * Check for host NMI, just to get that out of the way.
9878 */
9879 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9880 { /* normally likely */ }
9881 else
9882 {
9883 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9884 AssertRCReturn(rc2, rc2);
9885 uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
9886 if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
9887 return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
9888 }
9889
9890 /*
9891 * Check for single stepping event if we're stepping.
9892 */
9893 if (pVCpu->hm.s.fSingleInstruction)
9894 {
9895 switch (uExitReason)
9896 {
9897 case VMX_EXIT_MTF:
9898 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9899
9900 /* Various events: */
9901 case VMX_EXIT_XCPT_OR_NMI:
9902 case VMX_EXIT_EXT_INT:
9903 case VMX_EXIT_TRIPLE_FAULT:
9904 case VMX_EXIT_INT_WINDOW:
9905 case VMX_EXIT_NMI_WINDOW:
9906 case VMX_EXIT_TASK_SWITCH:
9907 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9908 case VMX_EXIT_APIC_ACCESS:
9909 case VMX_EXIT_EPT_VIOLATION:
9910 case VMX_EXIT_EPT_MISCONFIG:
9911 case VMX_EXIT_PREEMPT_TIMER:
9912
9913 /* Instruction specific VM-exits: */
9914 case VMX_EXIT_CPUID:
9915 case VMX_EXIT_GETSEC:
9916 case VMX_EXIT_HLT:
9917 case VMX_EXIT_INVD:
9918 case VMX_EXIT_INVLPG:
9919 case VMX_EXIT_RDPMC:
9920 case VMX_EXIT_RDTSC:
9921 case VMX_EXIT_RSM:
9922 case VMX_EXIT_VMCALL:
9923 case VMX_EXIT_VMCLEAR:
9924 case VMX_EXIT_VMLAUNCH:
9925 case VMX_EXIT_VMPTRLD:
9926 case VMX_EXIT_VMPTRST:
9927 case VMX_EXIT_VMREAD:
9928 case VMX_EXIT_VMRESUME:
9929 case VMX_EXIT_VMWRITE:
9930 case VMX_EXIT_VMXOFF:
9931 case VMX_EXIT_VMXON:
9932 case VMX_EXIT_MOV_CRX:
9933 case VMX_EXIT_MOV_DRX:
9934 case VMX_EXIT_IO_INSTR:
9935 case VMX_EXIT_RDMSR:
9936 case VMX_EXIT_WRMSR:
9937 case VMX_EXIT_MWAIT:
9938 case VMX_EXIT_MONITOR:
9939 case VMX_EXIT_PAUSE:
9940 case VMX_EXIT_GDTR_IDTR_ACCESS:
9941 case VMX_EXIT_LDTR_TR_ACCESS:
9942 case VMX_EXIT_INVEPT:
9943 case VMX_EXIT_RDTSCP:
9944 case VMX_EXIT_INVVPID:
9945 case VMX_EXIT_WBINVD:
9946 case VMX_EXIT_XSETBV:
9947 case VMX_EXIT_RDRAND:
9948 case VMX_EXIT_INVPCID:
9949 case VMX_EXIT_VMFUNC:
9950 case VMX_EXIT_RDSEED:
9951 case VMX_EXIT_XSAVES:
9952 case VMX_EXIT_XRSTORS:
9953 {
9954 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
9955 AssertRCReturn(rc, rc);
9956 if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
9957 || pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
9958 return VINF_EM_DBG_STEPPED;
9959 break;
9960 }
9961
9962 /* Errors and unexpected events: */
9963 case VMX_EXIT_INIT_SIGNAL:
9964 case VMX_EXIT_SIPI:
9965 case VMX_EXIT_IO_SMI:
9966 case VMX_EXIT_SMI:
9967 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9968 case VMX_EXIT_ERR_MSR_LOAD:
9969 case VMX_EXIT_ERR_MACHINE_CHECK:
9970 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9971 break;
9972
9973 default:
9974 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9975 break;
9976 }
9977 }
9978
9979 /*
9980 * Check for debugger event breakpoints and dtrace probes.
9981 */
9982 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9983 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9984 {
9985 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
9986 if (rcStrict != VINF_SUCCESS)
9987 return rcStrict;
9988 }
9989
9990 /*
9991 * Normal processing.
9992 */
9993#ifdef HMVMX_USE_FUNCTION_TABLE
9994 return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
9995#else
9996 return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
9997#endif
9998}
9999
10000
10001/**
10002 * Single steps guest code using VT-x.
10003 *
10004 * @returns Strict VBox status code (i.e. informational status codes too).
10005 * @param pVCpu The cross context virtual CPU structure.
10006 *
10007 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10008 */
10009static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu)
10010{
10011 VMXTRANSIENT VmxTransient;
10012 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10013
10014 /* Set HMCPU indicators. */
10015 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10016 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
10017 pVCpu->hm.s.fDebugWantRdTscExit = false;
10018 pVCpu->hm.s.fUsingDebugLoop = true;
10019
10020 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10021 VMXRUNDBGSTATE DbgState;
10022 hmR0VmxRunDebugStateInit(pVCpu, &DbgState);
10023 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
10024
10025 /*
10026 * The loop.
10027 */
10028 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10029 for (uint32_t cLoops = 0; ; cLoops++)
10030 {
10031 Assert(!HMR0SuspendPending());
10032 HMVMX_ASSERT_CPU_SAFE(pVCpu);
10033 bool fStepping = pVCpu->hm.s.fSingleInstruction;
10034
10035 /*
10036 * Preparatory work for running guest code, this may force us to return
10037 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10038 */
10039 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10040 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10041 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
10042 if (rcStrict != VINF_SUCCESS)
10043 break;
10044
10045 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10046 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10047
10048 /*
10049 * Now we can run the guest code.
10050 */
10051 int rcRun = hmR0VmxRunGuest(pVCpu);
10052
10053 /*
10054 * Restore any residual host-state and save any bits shared between host
10055 * and guest into the guest-CPU state. Re-enables interrupts!
10056 */
10057 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10058
10059 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10060 if (RT_SUCCESS(rcRun))
10061 { /* very likely */ }
10062 else
10063 {
10064 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10065 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10066 return rcRun;
10067 }
10068
10069 /* Profile the VM-exit. */
10070 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10071 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10072 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10073 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10074 HMVMX_START_EXIT_DISPATCH_PROF();
10075
10076 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10077
10078 /*
10079 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10080 */
10081 rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
10082 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10083 if (rcStrict != VINF_SUCCESS)
10084 break;
10085 if (cLoops > pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
10086 {
10087 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10088 rcStrict = VINF_EM_RAW_INTERRUPT;
10089 break;
10090 }
10091
10092 /*
10093 * Stepping: Did the RIP change, if so, consider it a single step.
10094 * Otherwise, make sure one of the TFs gets set.
10095 */
10096 if (fStepping)
10097 {
10098 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
10099 AssertRC(rc);
10100 if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
10101 || pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
10102 {
10103 rcStrict = VINF_EM_DBG_STEPPED;
10104 break;
10105 }
10106 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
10107 }
10108
10109 /*
10110 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10111 */
10112 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10113 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
10114 }
10115
10116 /*
10117 * Clear the X86_EFL_TF if necessary.
10118 */
10119 if (pVCpu->hm.s.fClearTrapFlag)
10120 {
10121 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
10122 AssertRC(rc);
10123 pVCpu->hm.s.fClearTrapFlag = false;
10124 pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
10125 }
10126 /** @todo there seems to be issues with the resume flag when the monitor trap
10127 * flag is pending without being used. Seen early in bios init when
10128 * accessing APIC page in protected mode. */
10129
10130 /*
10131 * Restore VM-exit control settings as we may not reenter this function the
10132 * next time around.
10133 */
10134 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10135
10136 /* Restore HMCPU indicators. */
10137 pVCpu->hm.s.fUsingDebugLoop = false;
10138 pVCpu->hm.s.fDebugWantRdTscExit = false;
10139 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10140
10141 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10142 return rcStrict;
10143}
10144
10145
10146/** @} */
10147
10148
10149/**
10150 * Checks if any expensive dtrace probes are enabled and we should go to the
10151 * debug loop.
10152 *
10153 * @returns true if we should use debug loop, false if not.
10154 */
10155static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10156{
10157 /* It's probably faster to OR the raw 32-bit counter variables together.
10158 Since the variables are in an array and the probes are next to one
10159 another (more or less), we have good locality. So, better read
10160 eight-nine cache lines ever time and only have one conditional, than
10161 128+ conditionals, right? */
10162 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10163 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10164 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10165 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10166 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10167 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10168 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10169 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10170 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10171 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10172 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10173 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10174 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10175 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10176 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10177 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10178 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10179 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10180 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10181 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10182 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10183 ) != 0
10184 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10185 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10186 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10187 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10188 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10189 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10190 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10191 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10192 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10193 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10194 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10195 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10196 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10197 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10198 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10199 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10200 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10201 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10202 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10203 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10204 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10205 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10206 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10207 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10208 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10209 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10210 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10211 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10212 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10213 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10214 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10215 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10216 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10217 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10218 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10219 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10220 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10221 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10222 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10223 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10224 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10225 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10226 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10227 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10228 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10229 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10230 ) != 0
10231 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10232 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10233 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10234 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10235 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10236 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10237 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10238 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10239 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10240 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10241 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10242 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10243 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10244 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10245 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10246 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10247 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10248 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10249 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10250 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10251 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10252 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10253 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10254 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10255 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10256 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10257 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10258 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10259 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10260 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10261 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10262 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10263 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10264 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10265 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10266 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10267 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10268 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10269 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10270 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10271 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10272 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10273 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10274 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10275 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10276 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10277 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10278 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10279 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10280 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10281 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10282 ) != 0;
10283}
10284
10285
10286/**
10287 * Runs the guest code using VT-x.
10288 *
10289 * @returns Strict VBox status code (i.e. informational status codes too).
10290 * @param pVCpu The cross context virtual CPU structure.
10291 */
10292VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
10293{
10294 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10295 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10296 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10297 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
10298
10299 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10300
10301 VBOXSTRICTRC rcStrict;
10302 if ( !pVCpu->hm.s.fUseDebugLoop
10303 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10304 && !DBGFIsStepping(pVCpu)
10305 && !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
10306 rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu);
10307 else
10308 rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu);
10309
10310 if (rcStrict == VERR_EM_INTERPRETER)
10311 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10312 else if (rcStrict == VINF_EM_RESET)
10313 rcStrict = VINF_EM_TRIPLE_FAULT;
10314
10315 int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
10316 if (RT_FAILURE(rc2))
10317 {
10318 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10319 rcStrict = rc2;
10320 }
10321 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10322 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10323 return rcStrict;
10324}
10325
10326
10327#ifndef HMVMX_USE_FUNCTION_TABLE
10328DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10329{
10330#ifdef DEBUG_ramshankar
10331#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10332 do { \
10333 if (a_fSave != 0) \
10334 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10335 VBOXSTRICTRC rcStrict = a_CallExpr; \
10336 if (a_fSave != 0) \
10337 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10338 return rcStrict; \
10339 } while (0)
10340#else
10341# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10342#endif
10343 switch (rcReason)
10344 {
10345 case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
10346 case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
10347 case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
10348 case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
10349 case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
10350 case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
10351 case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
10352 case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
10353 case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
10354 case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
10355 case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
10356 case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
10357 case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
10358 case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
10359 case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
10360 case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
10361 case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
10362 case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
10363 case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
10364 case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
10365 case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
10366 case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
10367 case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
10368 case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pVmxTransient));
10369 case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
10370 case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
10371 case VMX_EXIT_GDTR_IDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10372 case VMX_EXIT_LDTR_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10373 case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
10374 case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
10375 case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
10376 case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
10377 case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
10378 case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
10379
10380 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
10381 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
10382 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pVmxTransient);
10383 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pVmxTransient);
10384 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pVmxTransient);
10385 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pVmxTransient);
10386 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pVmxTransient);
10387 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
10388 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pVmxTransient);
10389
10390 case VMX_EXIT_VMCLEAR:
10391 case VMX_EXIT_VMLAUNCH:
10392 case VMX_EXIT_VMPTRLD:
10393 case VMX_EXIT_VMPTRST:
10394 case VMX_EXIT_VMREAD:
10395 case VMX_EXIT_VMRESUME:
10396 case VMX_EXIT_VMWRITE:
10397 case VMX_EXIT_VMXOFF:
10398 case VMX_EXIT_VMXON:
10399 case VMX_EXIT_INVEPT:
10400 case VMX_EXIT_INVVPID:
10401 case VMX_EXIT_VMFUNC:
10402 case VMX_EXIT_XSAVES:
10403 case VMX_EXIT_XRSTORS:
10404 return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
10405
10406 case VMX_EXIT_ENCLS:
10407 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10408 case VMX_EXIT_PML_FULL:
10409 default:
10410 return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
10411 }
10412#undef VMEXIT_CALL_RET
10413}
10414#endif /* !HMVMX_USE_FUNCTION_TABLE */
10415
10416
10417#ifdef VBOX_STRICT
10418/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10419# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10420 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10421
10422# define HMVMX_ASSERT_PREEMPT_CPUID() \
10423 do { \
10424 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10425 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10426 } while (0)
10427
10428# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10429 do { \
10430 AssertPtr((a_pVCpu)); \
10431 AssertPtr((a_pVmxTransient)); \
10432 Assert((a_pVmxTransient)->fVMEntryFailed == false); \
10433 Assert(ASMIntAreEnabled()); \
10434 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10435 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10436 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
10437 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10438 if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
10439 HMVMX_ASSERT_PREEMPT_CPUID(); \
10440 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10441 } while (0)
10442
10443# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10444 do { \
10445 Log4Func(("\n")); \
10446 } while (0)
10447#else
10448# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10449 do { \
10450 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10451 NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
10452 } while (0)
10453# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
10454#endif
10455
10456
10457/**
10458 * Advances the guest RIP by the specified number of bytes.
10459 *
10460 * @param pVCpu The cross context virtual CPU structure.
10461 * @param cbInstr Number of bytes to advance the RIP by.
10462 *
10463 * @remarks No-long-jump zone!!!
10464 */
10465DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
10466{
10467 /* Advance the RIP. */
10468 pVCpu->cpum.GstCtx.rip += cbInstr;
10469 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10470
10471 /* Update interrupt inhibition. */
10472 if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10473 && pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
10474 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10475}
10476
10477
10478/**
10479 * Advances the guest RIP after reading it from the VMCS.
10480 *
10481 * @returns VBox status code, no informational status codes.
10482 * @param pVCpu The cross context virtual CPU structure.
10483 * @param pVmxTransient Pointer to the VMX transient structure.
10484 *
10485 * @remarks No-long-jump zone!!!
10486 */
10487static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10488{
10489 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10490 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS);
10491 AssertRCReturn(rc, rc);
10492
10493 hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
10494
10495 /*
10496 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10497 * pending debug exception field as it takes care of priority of events.
10498 *
10499 * See Intel spec. 32.2.1 "Debug Exceptions".
10500 */
10501 if ( !pVCpu->hm.s.fSingleInstruction
10502 && pVCpu->cpum.GstCtx.eflags.Bits.u1TF)
10503 {
10504 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10505 AssertRCReturn(rc, rc);
10506 }
10507
10508 return VINF_SUCCESS;
10509}
10510
10511
10512/**
10513 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10514 * and update error record fields accordingly.
10515 *
10516 * @return VMX_IGS_* return codes.
10517 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10518 * wrong with the guest state.
10519 *
10520 * @param pVCpu The cross context virtual CPU structure.
10521 *
10522 * @remarks This function assumes our cache of the VMCS controls
10523 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10524 */
10525static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu)
10526{
10527#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10528#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10529 uError = (err); \
10530 break; \
10531 } else do { } while (0)
10532
10533 int rc;
10534 PVM pVM = pVCpu->CTX_SUFF(pVM);
10535 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10536 uint32_t uError = VMX_IGS_ERROR;
10537 uint32_t u32Val;
10538 bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10539
10540 do
10541 {
10542 /*
10543 * CR0.
10544 */
10545 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10546 uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10547 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10548 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10549 if (fUnrestrictedGuest)
10550 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
10551
10552 uint32_t u32GuestCr0;
10553 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10554 AssertRCBreak(rc);
10555 HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10556 HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10557 if ( !fUnrestrictedGuest
10558 && (u32GuestCr0 & X86_CR0_PG)
10559 && !(u32GuestCr0 & X86_CR0_PE))
10560 {
10561 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10562 }
10563
10564 /*
10565 * CR4.
10566 */
10567 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10568 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10569
10570 uint32_t u32GuestCr4;
10571 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10572 AssertRCBreak(rc);
10573 HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10574 HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10575
10576 /*
10577 * IA32_DEBUGCTL MSR.
10578 */
10579 uint64_t u64Val;
10580 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10581 AssertRCBreak(rc);
10582 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10583 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10584 {
10585 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10586 }
10587 uint64_t u64DebugCtlMsr = u64Val;
10588
10589#ifdef VBOX_STRICT
10590 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10591 AssertRCBreak(rc);
10592 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10593#endif
10594 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
10595
10596 /*
10597 * RIP and RFLAGS.
10598 */
10599 uint32_t u32Eflags;
10600#if HC_ARCH_BITS == 64
10601 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10602 AssertRCBreak(rc);
10603 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10604 if ( !fLongModeGuest
10605 || !pCtx->cs.Attr.n.u1Long)
10606 {
10607 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10608 }
10609 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10610 * must be identical if the "IA-32e mode guest" VM-entry
10611 * control is 1 and CS.L is 1. No check applies if the
10612 * CPU supports 64 linear-address bits. */
10613
10614 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10615 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10616 AssertRCBreak(rc);
10617 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10618 VMX_IGS_RFLAGS_RESERVED);
10619 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10620 u32Eflags = u64Val;
10621#else
10622 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10623 AssertRCBreak(rc);
10624 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10625 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10626#endif
10627
10628 if ( fLongModeGuest
10629 || ( fUnrestrictedGuest
10630 && !(u32GuestCr0 & X86_CR0_PE)))
10631 {
10632 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10633 }
10634
10635 uint32_t u32EntryInfo;
10636 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10637 AssertRCBreak(rc);
10638 if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
10639 && VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
10640 {
10641 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10642 }
10643
10644 /*
10645 * 64-bit checks.
10646 */
10647#if HC_ARCH_BITS == 64
10648 if (fLongModeGuest)
10649 {
10650 HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10651 HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10652 }
10653
10654 if ( !fLongModeGuest
10655 && (u32GuestCr4 & X86_CR4_PCIDE))
10656 {
10657 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10658 }
10659
10660 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10661 * 51:32 beyond the processor's physical-address width are 0. */
10662
10663 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10664 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10665 {
10666 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10667 }
10668
10669 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10670 AssertRCBreak(rc);
10671 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10672
10673 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10674 AssertRCBreak(rc);
10675 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10676#endif
10677
10678 /*
10679 * PERF_GLOBAL MSR.
10680 */
10681 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
10682 {
10683 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10684 AssertRCBreak(rc);
10685 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10686 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10687 }
10688
10689 /*
10690 * PAT MSR.
10691 */
10692 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
10693 {
10694 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10695 AssertRCBreak(rc);
10696 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10697 for (unsigned i = 0; i < 8; i++)
10698 {
10699 uint8_t u8Val = (u64Val & 0xff);
10700 if ( u8Val != 0 /* UC */
10701 && u8Val != 1 /* WC */
10702 && u8Val != 4 /* WT */
10703 && u8Val != 5 /* WP */
10704 && u8Val != 6 /* WB */
10705 && u8Val != 7 /* UC- */)
10706 {
10707 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10708 }
10709 u64Val >>= 8;
10710 }
10711 }
10712
10713 /*
10714 * EFER MSR.
10715 */
10716 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
10717 {
10718 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10719 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10720 AssertRCBreak(rc);
10721 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10722 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10723 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10724 & VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
10725 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10726 /** @todo r=ramshankar: Unrestricted check here is probably wrong, see
10727 * iemVmxVmentryCheckGuestState(). */
10728 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10729 || !(u32GuestCr0 & X86_CR0_PG)
10730 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10731 VMX_IGS_EFER_LMA_LME_MISMATCH);
10732 }
10733
10734 /*
10735 * Segment registers.
10736 */
10737 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10738 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10739 if (!(u32Eflags & X86_EFL_VM))
10740 {
10741 /* CS */
10742 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10743 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10744 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10745 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10746 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10747 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10748 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10749 /* CS cannot be loaded with NULL in protected mode. */
10750 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10751 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10752 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10753 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10754 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10755 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10756 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10757 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10758 else
10759 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10760
10761 /* SS */
10762 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10763 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10764 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10765 if ( !(pCtx->cr0 & X86_CR0_PE)
10766 || pCtx->cs.Attr.n.u4Type == 3)
10767 {
10768 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10769 }
10770 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10771 {
10772 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10773 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10774 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10775 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10776 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10777 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10778 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10779 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10780 }
10781
10782 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmenReg(). */
10783 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10784 {
10785 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10786 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10787 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10788 || pCtx->ds.Attr.n.u4Type > 11
10789 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10790 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10791 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10792 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10793 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10794 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10795 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10796 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10797 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10798 }
10799 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10800 {
10801 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10802 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10803 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10804 || pCtx->es.Attr.n.u4Type > 11
10805 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10806 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10807 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10808 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10809 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10810 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10811 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10812 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10813 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10814 }
10815 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10816 {
10817 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10818 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10819 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10820 || pCtx->fs.Attr.n.u4Type > 11
10821 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10822 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10823 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10824 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10825 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10826 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10827 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10828 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10829 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10830 }
10831 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10832 {
10833 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10834 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10835 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10836 || pCtx->gs.Attr.n.u4Type > 11
10837 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10838 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10839 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10840 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10841 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10842 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10843 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10844 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10845 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10846 }
10847 /* 64-bit capable CPUs. */
10848#if HC_ARCH_BITS == 64
10849 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10850 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10851 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10852 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10853 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10854 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10855 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10856 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10857 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10858 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10859 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10860#endif
10861 }
10862 else
10863 {
10864 /* V86 mode checks. */
10865 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10866 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10867 {
10868 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10869 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10870 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10871 }
10872 else
10873 {
10874 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10875 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10876 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10877 }
10878
10879 /* CS */
10880 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10881 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10882 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10883 /* SS */
10884 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10885 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10886 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10887 /* DS */
10888 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10889 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10890 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10891 /* ES */
10892 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10893 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10894 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10895 /* FS */
10896 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10897 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10898 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10899 /* GS */
10900 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10901 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10902 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10903 /* 64-bit capable CPUs. */
10904#if HC_ARCH_BITS == 64
10905 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10906 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10907 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10908 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10909 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10910 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10911 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10912 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10913 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10914 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10915 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10916#endif
10917 }
10918
10919 /*
10920 * TR.
10921 */
10922 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10923 /* 64-bit capable CPUs. */
10924#if HC_ARCH_BITS == 64
10925 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10926#endif
10927 if (fLongModeGuest)
10928 {
10929 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10930 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10931 }
10932 else
10933 {
10934 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10935 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10936 VMX_IGS_TR_ATTR_TYPE_INVALID);
10937 }
10938 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10939 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10940 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10941 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10942 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10943 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10944 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10945 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10946
10947 /*
10948 * GDTR and IDTR.
10949 */
10950#if HC_ARCH_BITS == 64
10951 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10952 AssertRCBreak(rc);
10953 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10954
10955 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10956 AssertRCBreak(rc);
10957 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10958#endif
10959
10960 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10961 AssertRCBreak(rc);
10962 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10963
10964 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10965 AssertRCBreak(rc);
10966 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10967
10968 /*
10969 * Guest Non-Register State.
10970 */
10971 /* Activity State. */
10972 uint32_t u32ActivityState;
10973 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10974 AssertRCBreak(rc);
10975 HMVMX_CHECK_BREAK( !u32ActivityState
10976 || (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
10977 VMX_IGS_ACTIVITY_STATE_INVALID);
10978 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10979 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10980 uint32_t u32IntrState;
10981 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
10982 AssertRCBreak(rc);
10983 if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
10984 || u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10985 {
10986 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10987 }
10988
10989 /** @todo Activity state and injecting interrupts. Left as a todo since we
10990 * currently don't use activity states but ACTIVE. */
10991
10992 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10993 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10994
10995 /* Guest interruptibility-state. */
10996 HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10997 HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
10998 != (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10999 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11000 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11001 || !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
11002 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11003 if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
11004 {
11005 if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
11006 {
11007 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
11008 && !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
11009 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11010 }
11011 else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
11012 {
11013 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
11014 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11015 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
11016 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11017 }
11018 }
11019 /** @todo Assumes the processor is not in SMM. */
11020 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
11021 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11022 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
11023 || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
11024 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11025 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
11026 && VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
11027 && VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
11028 {
11029 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
11030 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11031 }
11032
11033 /* Pending debug exceptions. */
11034#if HC_ARCH_BITS == 64
11035 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
11036 AssertRCBreak(rc);
11037 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11038 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11039 u32Val = u64Val; /* For pending debug exceptions checks below. */
11040#else
11041 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
11042 AssertRCBreak(rc);
11043 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11044 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11045#endif
11046
11047 if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
11048 || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
11049 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11050 {
11051 if ( (u32Eflags & X86_EFL_TF)
11052 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11053 {
11054 /* Bit 14 is PendingDebug.BS. */
11055 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11056 }
11057 if ( !(u32Eflags & X86_EFL_TF)
11058 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11059 {
11060 /* Bit 14 is PendingDebug.BS. */
11061 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11062 }
11063 }
11064
11065 /* VMCS link pointer. */
11066 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11067 AssertRCBreak(rc);
11068 if (u64Val != UINT64_C(0xffffffffffffffff))
11069 {
11070 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11071 /** @todo Bits beyond the processor's physical-address width MBZ. */
11072 /** @todo 32-bit located in memory referenced by value of this field (as a
11073 * physical address) must contain the processor's VMCS revision ID. */
11074 /** @todo SMM checks. */
11075 }
11076
11077 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11078 * not using Nested Paging? */
11079 if ( pVM->hm.s.fNestedPaging
11080 && !fLongModeGuest
11081 && CPUMIsGuestInPAEModeEx(pCtx))
11082 {
11083 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11084 AssertRCBreak(rc);
11085 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11086
11087 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11088 AssertRCBreak(rc);
11089 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11090
11091 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11092 AssertRCBreak(rc);
11093 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11094
11095 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11096 AssertRCBreak(rc);
11097 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11098 }
11099
11100 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11101 if (uError == VMX_IGS_ERROR)
11102 uError = VMX_IGS_REASON_NOT_FOUND;
11103 } while (0);
11104
11105 pVCpu->hm.s.u32HMError = uError;
11106 return uError;
11107
11108#undef HMVMX_ERROR_BREAK
11109#undef HMVMX_CHECK_BREAK
11110}
11111
11112
11113/** @name VM-exit handlers.
11114 * @{
11115 */
11116/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11117/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11118/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11119
11120/**
11121 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11122 */
11123HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11124{
11125 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11126 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11127 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11128 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11129 return VINF_SUCCESS;
11130 return VINF_EM_RAW_INTERRUPT;
11131}
11132
11133
11134/**
11135 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11136 */
11137HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11138{
11139 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11140 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11141
11142 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11143 AssertRCReturn(rc, rc);
11144
11145 uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
11146 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
11147 && uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
11148 Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11149
11150 if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
11151 {
11152 /*
11153 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
11154 * injected it ourselves and anything we inject is not going to cause a VM-exit directly
11155 * for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
11156 *
11157 * [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11158 * [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
11159 */
11160 VMXDispatchHostNmi();
11161 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11162 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11163 return VINF_SUCCESS;
11164 }
11165
11166 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11167 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
11168 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11169 { /* likely */ }
11170 else
11171 {
11172 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11173 rcStrictRc1 = VINF_SUCCESS;
11174 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11175 return rcStrictRc1;
11176 }
11177
11178 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11179 uint32_t uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
11180 switch (uIntType)
11181 {
11182 case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11183 Assert(uVector == X86_XCPT_DB);
11184 RT_FALL_THRU();
11185 case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11186 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
11187 RT_FALL_THRU();
11188 case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11189 {
11190 /*
11191 * If there's any exception caused as a result of event injection, the resulting
11192 * secondary/final execption will be pending, we shall continue guest execution
11193 * after injecting the event. The page-fault case is complicated and we manually
11194 * handle any currently pending event in hmR0VmxExitXcptPF.
11195 */
11196 if (!pVCpu->hm.s.Event.fPending)
11197 { /* likely */ }
11198 else if (uVector != X86_XCPT_PF)
11199 {
11200 rc = VINF_SUCCESS;
11201 break;
11202 }
11203
11204 switch (uVector)
11205 {
11206 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
11207 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
11208 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
11209 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
11210 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
11211 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
11212
11213 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11214 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11215 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11216 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11217 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11218 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11219 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11220 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11221 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11222 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11223 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11224 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11225 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11226 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11227 default:
11228 {
11229 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11230 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11231 {
11232 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11233 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11234 Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
11235
11236 rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0);
11237 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11238 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11239 AssertRCReturn(rc, rc);
11240 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11241 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11242 0 /* GCPtrFaultAddress */);
11243 }
11244 else
11245 {
11246 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11247 pVCpu->hm.s.u32HMError = uVector;
11248 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11249 }
11250 break;
11251 }
11252 }
11253 break;
11254 }
11255
11256 default:
11257 {
11258 pVCpu->hm.s.u32HMError = uExitIntInfo;
11259 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11260 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
11261 break;
11262 }
11263 }
11264 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11265 return rc;
11266}
11267
11268
11269/**
11270 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11271 */
11272HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11273{
11274 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11275
11276 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11277 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11278
11279 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11280 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11281 return VINF_SUCCESS;
11282}
11283
11284
11285/**
11286 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11287 */
11288HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11289{
11290 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11291 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)))
11292 {
11293 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11294 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11295 }
11296
11297 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS));
11298
11299 /*
11300 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11301 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11302 */
11303 uint32_t fIntrState = 0;
11304 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
11305 AssertRCReturn(rc, rc);
11306
11307 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
11308 if ( fBlockSti
11309 && VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11310 {
11311 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11312 }
11313
11314 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11315 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11316
11317 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11318 return VINF_SUCCESS;
11319}
11320
11321
11322/**
11323 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11324 */
11325HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11326{
11327 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11328 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11329}
11330
11331
11332/**
11333 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11334 */
11335HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11336{
11337 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11338 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11339}
11340
11341
11342/**
11343 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11344 */
11345HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11346{
11347 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11348
11349 /*
11350 * Get the state we need and update the exit history entry.
11351 */
11352 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11353 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
11354 AssertRCReturn(rc, rc);
11355
11356 VBOXSTRICTRC rcStrict;
11357 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11358 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
11359 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11360 if (!pExitRec)
11361 {
11362 /*
11363 * Regular CPUID instruction execution.
11364 */
11365 rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
11366 if (rcStrict == VINF_SUCCESS)
11367 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11368 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11369 {
11370 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11371 rcStrict = VINF_SUCCESS;
11372 }
11373 }
11374 else
11375 {
11376 /*
11377 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
11378 */
11379 int rc2 = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11380 AssertRCReturn(rc2, rc2);
11381
11382 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11383 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11384
11385 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11386 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11387
11388 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11389 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11390 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11391 }
11392 return rcStrict;
11393}
11394
11395
11396/**
11397 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11398 */
11399HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11400{
11401 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11402 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR4);
11403 AssertRCReturn(rc, rc);
11404
11405 if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
11406 return VINF_EM_RAW_EMULATE_INSTR;
11407
11408 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11409 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11410}
11411
11412
11413/**
11414 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11415 */
11416HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11417{
11418 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11419 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11420 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11421 AssertRCReturn(rc, rc);
11422
11423 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11424 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11425 {
11426 /* If we get a spurious VM-exit when offsetting is enabled,
11427 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11428 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
11429 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11430 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11431 }
11432 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11433 {
11434 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11435 rcStrict = VINF_SUCCESS;
11436 }
11437 return rcStrict;
11438}
11439
11440
11441/**
11442 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11443 */
11444HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11445{
11446 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11447 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_TSC_AUX);
11448 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11449 AssertRCReturn(rc, rc);
11450
11451 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11452 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11453 {
11454 /* If we get a spurious VM-exit when offsetting is enabled,
11455 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11456 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
11457 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11458 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11459 }
11460 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11461 {
11462 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11463 rcStrict = VINF_SUCCESS;
11464 }
11465 return rcStrict;
11466}
11467
11468
11469/**
11470 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11471 */
11472HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11473{
11474 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11475 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11476 AssertRCReturn(rc, rc);
11477
11478 PVM pVM = pVCpu->CTX_SUFF(pVM);
11479 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11480 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11481 if (RT_LIKELY(rc == VINF_SUCCESS))
11482 {
11483 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11484 Assert(pVmxTransient->cbInstr == 2);
11485 }
11486 else
11487 {
11488 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11489 rc = VERR_EM_INTERPRETER;
11490 }
11491 return rc;
11492}
11493
11494
11495/**
11496 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11497 */
11498HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11499{
11500 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11501
11502 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11503 if (EMAreHypercallInstructionsEnabled(pVCpu))
11504 {
11505 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CR0
11506 | CPUMCTX_EXTRN_SS | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_EFER);
11507 AssertRCReturn(rc, rc);
11508
11509 /* Perform the hypercall. */
11510 rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
11511 if (rcStrict == VINF_SUCCESS)
11512 {
11513 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11514 AssertRCReturn(rc, rc);
11515 }
11516 else
11517 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11518 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11519 || RT_FAILURE(rcStrict));
11520
11521 /* If the hypercall changes anything other than guest's general-purpose registers,
11522 we would need to reload the guest changed bits here before VM-entry. */
11523 }
11524 else
11525 Log4Func(("Hypercalls not enabled\n"));
11526
11527 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11528 if (RT_FAILURE(rcStrict))
11529 {
11530 hmR0VmxSetPendingXcptUD(pVCpu);
11531 rcStrict = VINF_SUCCESS;
11532 }
11533
11534 return rcStrict;
11535}
11536
11537
11538/**
11539 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11540 */
11541HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11542{
11543 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11544 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11545
11546 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11547 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11548 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
11549 AssertRCReturn(rc, rc);
11550
11551 VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
11552
11553 if (rcStrict == VINF_SUCCESS || rcStrict == VINF_PGM_SYNC_CR3)
11554 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11555 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11556 {
11557 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11558 rcStrict = VINF_SUCCESS;
11559 }
11560 else
11561 AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) sttus: %Rrc\n", pVmxTransient->uExitQual,
11562 VBOXSTRICTRC_VAL(rcStrict)));
11563 return rcStrict;
11564}
11565
11566
11567/**
11568 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11569 */
11570HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11571{
11572 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11573 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11574 AssertRCReturn(rc, rc);
11575
11576 PVM pVM = pVCpu->CTX_SUFF(pVM);
11577 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11578 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11579 if (RT_LIKELY(rc == VINF_SUCCESS))
11580 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11581 else
11582 {
11583 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11584 rc = VERR_EM_INTERPRETER;
11585 }
11586 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11587 return rc;
11588}
11589
11590
11591/**
11592 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11593 */
11594HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11595{
11596 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11597 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11598 AssertRCReturn(rc, rc);
11599
11600 PVM pVM = pVCpu->CTX_SUFF(pVM);
11601 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11602 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11603 rc = VBOXSTRICTRC_VAL(rc2);
11604 if (RT_LIKELY( rc == VINF_SUCCESS
11605 || rc == VINF_EM_HALT))
11606 {
11607 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11608 AssertRCReturn(rc3, rc3);
11609
11610 if ( rc == VINF_EM_HALT
11611 && EMMonitorWaitShouldContinue(pVCpu, pCtx))
11612 rc = VINF_SUCCESS;
11613 }
11614 else
11615 {
11616 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11617 rc = VERR_EM_INTERPRETER;
11618 }
11619 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11620 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11621 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11622 return rc;
11623}
11624
11625
11626/**
11627 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11628 */
11629HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11630{
11631 /*
11632 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11633 * mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11634 * treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11635 * VMX root operation. If we get here, something funny is going on.
11636 *
11637 * See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11638 */
11639 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11640 AssertMsgFailed(("Unexpected RSM VM-exit\n"));
11641 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11642}
11643
11644
11645/**
11646 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11647 */
11648HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11649{
11650 /*
11651 * This can only happen if we support dual-monitor treatment of SMI, which can be activated
11652 * by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11653 * this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11654 * an SMI. If we get here, something funny is going on.
11655 *
11656 * See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11657 * See Intel spec. 25.3 "Other Causes of VM-Exits"
11658 */
11659 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11660 AssertMsgFailed(("Unexpected SMI VM-exit\n"));
11661 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11662}
11663
11664
11665/**
11666 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11667 */
11668HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11669{
11670 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11671 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11672 AssertMsgFailed(("Unexpected IO SMI VM-exit\n"));
11673 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11674}
11675
11676
11677/**
11678 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11679 */
11680HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11681{
11682 /*
11683 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11684 * We don't make use of it as our guests don't have direct access to the host LAPIC.
11685 * See Intel spec. 25.3 "Other Causes of VM-exits".
11686 */
11687 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11688 AssertMsgFailed(("Unexpected SIPI VM-exit\n"));
11689 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11690}
11691
11692
11693/**
11694 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11695 * VM-exit.
11696 */
11697HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11698{
11699 /*
11700 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11701 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11702 *
11703 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11704 * See Intel spec. "23.8 Restrictions on VMX operation".
11705 */
11706 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11707 return VINF_SUCCESS;
11708}
11709
11710
11711/**
11712 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11713 * VM-exit.
11714 */
11715HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11716{
11717 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11718 return VINF_EM_RESET;
11719}
11720
11721
11722/**
11723 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11724 */
11725HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11726{
11727 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11728 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_HLT_EXIT);
11729
11730 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11731 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RFLAGS);
11732 AssertRCReturn(rc, rc);
11733
11734 if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
11735 rc = VINF_SUCCESS;
11736 else
11737 rc = VINF_EM_HALT;
11738
11739 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11740 if (rc != VINF_SUCCESS)
11741 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11742 return rc;
11743}
11744
11745
11746/**
11747 * VM-exit handler for instructions that result in a \#UD exception delivered to
11748 * the guest.
11749 */
11750HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11751{
11752 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11753 hmR0VmxSetPendingXcptUD(pVCpu);
11754 return VINF_SUCCESS;
11755}
11756
11757
11758/**
11759 * VM-exit handler for expiry of the VMX preemption timer.
11760 */
11761HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11762{
11763 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11764
11765 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11766 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11767
11768 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11769 PVM pVM = pVCpu->CTX_SUFF(pVM);
11770 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11771 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11772 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11773}
11774
11775
11776/**
11777 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11778 */
11779HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11780{
11781 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11782
11783 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11784 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_CR4);
11785 AssertRCReturn(rc, rc);
11786
11787 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11788 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11789 : HM_CHANGED_RAISED_XCPT_MASK);
11790
11791 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11792 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
11793
11794 return rcStrict;
11795}
11796
11797
11798/**
11799 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11800 */
11801HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11802{
11803 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11804 /** @todo Use VM-exit instruction information. */
11805 return VERR_EM_INTERPRETER;
11806}
11807
11808
11809/**
11810 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11811 * Error VM-exit.
11812 */
11813HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11814{
11815 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11816 AssertRCReturn(rc, rc);
11817 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11818 if (RT_FAILURE(rc))
11819 return rc;
11820
11821 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
11822 NOREF(uInvalidReason);
11823
11824#ifdef VBOX_STRICT
11825 uint32_t fIntrState;
11826 RTHCUINTREG uHCReg;
11827 uint64_t u64Val;
11828 uint32_t u32Val;
11829
11830 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11831 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11832 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11833 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
11834 AssertRCReturn(rc, rc);
11835
11836 Log4(("uInvalidReason %u\n", uInvalidReason));
11837 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11838 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11839 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11840 Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
11841
11842 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11843 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11844 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11845 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11846 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11847 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11848 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11849 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11850 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11851 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11852 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11853 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11854
11855 hmR0DumpRegs(pVCpu);
11856#else
11857 NOREF(pVmxTransient);
11858#endif
11859
11860 return VERR_VMX_INVALID_GUEST_STATE;
11861}
11862
11863
11864/**
11865 * VM-exit handler for VM-entry failure due to an MSR-load
11866 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11867 */
11868HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11869{
11870 AssertMsgFailed(("Unexpected MSR-load exit\n"));
11871 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11872}
11873
11874
11875/**
11876 * VM-exit handler for VM-entry failure due to a machine-check event
11877 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11878 */
11879HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11880{
11881 AssertMsgFailed(("Unexpected machine-check event exit\n"));
11882 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11883}
11884
11885
11886/**
11887 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11888 * theory.
11889 */
11890HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11891{
11892 RT_NOREF2(pVCpu, pVmxTransient);
11893 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d\n", pVmxTransient->uExitReason));
11894 return VERR_VMX_UNDEFINED_EXIT_CODE;
11895}
11896
11897
11898/**
11899 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11900 * (VMX_EXIT_GDTR_IDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11901 * Conditional VM-exit.
11902 */
11903HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11904{
11905 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11906
11907 /* By default, we don't enable VMX_PROC_CTLS2_DESCRIPTOR_TABLE_EXIT. */
11908 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11909 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_DESC_TABLE_EXIT)
11910 return VERR_EM_INTERPRETER;
11911 AssertMsgFailed(("Unexpected XDTR access\n"));
11912 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11913}
11914
11915
11916/**
11917 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11918 */
11919HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11920{
11921 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11922
11923 /* By default, we don't enable VMX_PROC_CTLS2_RDRAND_EXIT. */
11924 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_RDRAND_EXIT)
11925 return VERR_EM_INTERPRETER;
11926 AssertMsgFailed(("Unexpected RDRAND exit\n"));
11927 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11928}
11929
11930
11931/**
11932 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11933 */
11934HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11935{
11936 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11937
11938 /** @todo Optimize this: We currently drag in in the whole MSR state
11939 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11940 * MSRs required. That would require changes to IEM and possibly CPUM too.
11941 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11942 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
11943 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11944 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
11945 switch (idMsr)
11946 {
11947 /* The FS and GS base MSRs are not part of the above all-MSRs mask. */
11948 case MSR_K8_FS_BASE: rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_FS); break;
11949 case MSR_K8_GS_BASE: rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_GS); break;
11950 }
11951 AssertRCReturn(rc, rc);
11952
11953 Log4Func(("ecx=%#RX32\n", idMsr));
11954
11955#ifdef VBOX_STRICT
11956 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
11957 {
11958 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr)
11959 && idMsr != MSR_K6_EFER)
11960 {
11961 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
11962 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11963 }
11964 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11965 {
11966 VMXMSREXITREAD enmRead;
11967 VMXMSREXITWRITE enmWrite;
11968 int rc2 = HMVmxGetMsrPermission(pVCpu->hm.s.vmx.pvMsrBitmap, idMsr, &enmRead, &enmWrite);
11969 AssertRCReturn(rc2, rc2);
11970 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11971 {
11972 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
11973 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11974 }
11975 }
11976 }
11977#endif
11978
11979 VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
11980 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11981 if (rcStrict == VINF_SUCCESS)
11982 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11983 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX);
11984 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11985 {
11986 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11987 rcStrict = VINF_SUCCESS;
11988 }
11989 else
11990 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11991
11992 return rcStrict;
11993}
11994
11995
11996/**
11997 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11998 */
11999HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12000{
12001 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12002
12003 /** @todo Optimize this: We currently drag in in the whole MSR state
12004 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
12005 * MSRs required. That would require changes to IEM and possibly CPUM too.
12006 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
12007 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
12008 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12009 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK
12010 | CPUMCTX_EXTRN_ALL_MSRS);
12011 switch (idMsr)
12012 {
12013 /*
12014 * The FS and GS base MSRs are not part of the above all-MSRs mask.
12015 *
12016 * Although we don't need to fetch the base as it will be overwritten shortly, while
12017 * loading guest-state we would also load the entire segment register including limit
12018 * and attributes and thus we need to load them here.
12019 */
12020 case MSR_K8_FS_BASE: rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_FS); break;
12021 case MSR_K8_GS_BASE: rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_GS); break;
12022 }
12023 AssertRCReturn(rc, rc);
12024
12025 Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
12026
12027 VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
12028 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12029
12030 if (rcStrict == VINF_SUCCESS)
12031 {
12032 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12033
12034 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
12035 if ( idMsr == MSR_IA32_APICBASE
12036 || ( idMsr >= MSR_IA32_X2APIC_START
12037 && idMsr <= MSR_IA32_X2APIC_END))
12038 {
12039 /*
12040 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12041 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before IEM changes it.
12042 */
12043 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
12044 }
12045 else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12046 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12047 else if (idMsr == MSR_K6_EFER)
12048 {
12049 /*
12050 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12051 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12052 * the other bits as well, SCE and NXE. See @bugref{7368}.
12053 */
12054 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS
12055 | HM_CHANGED_VMX_EXIT_CTLS);
12056 }
12057
12058 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12059 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
12060 {
12061 switch (idMsr)
12062 {
12063 case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
12064 case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
12065 case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
12066 case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
12067 case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
12068 case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
12069 default:
12070 {
12071 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
12072 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12073 else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
12074 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
12075 break;
12076 }
12077 }
12078 }
12079#ifdef VBOX_STRICT
12080 else
12081 {
12082 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12083 switch (idMsr)
12084 {
12085 case MSR_IA32_SYSENTER_CS:
12086 case MSR_IA32_SYSENTER_EIP:
12087 case MSR_IA32_SYSENTER_ESP:
12088 case MSR_K8_FS_BASE:
12089 case MSR_K8_GS_BASE:
12090 {
12091 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
12092 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12093 }
12094
12095 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12096 default:
12097 {
12098 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
12099 {
12100 /* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
12101 if (idMsr != MSR_K6_EFER)
12102 {
12103 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12104 idMsr));
12105 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12106 }
12107 }
12108
12109 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
12110 {
12111 VMXMSREXITREAD enmRead;
12112 VMXMSREXITWRITE enmWrite;
12113 int rc2 = HMVmxGetMsrPermission(pVCpu->hm.s.vmx.pvMsrBitmap, idMsr, &enmRead, &enmWrite);
12114 AssertRCReturn(rc2, rc2);
12115 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12116 {
12117 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
12118 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12119 }
12120 }
12121 break;
12122 }
12123 }
12124 }
12125#endif /* VBOX_STRICT */
12126 }
12127 else if (rcStrict == VINF_IEM_RAISED_XCPT)
12128 {
12129 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
12130 rcStrict = VINF_SUCCESS;
12131 }
12132 else
12133 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12134
12135 return rcStrict;
12136}
12137
12138
12139/**
12140 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12141 */
12142HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12143{
12144 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12145 /** @todo The guest has likely hit a contended spinlock. We might want to
12146 * poke a schedule different guest VCPU. */
12147 return VINF_EM_RAW_INTERRUPT;
12148}
12149
12150
12151/**
12152 * VM-exit handler for when the TPR value is lowered below the specified
12153 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12154 */
12155HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12156{
12157 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12158 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
12159
12160 /*
12161 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12162 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12163 */
12164 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12165 return VINF_SUCCESS;
12166}
12167
12168
12169/**
12170 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12171 * VM-exit.
12172 *
12173 * @retval VINF_SUCCESS when guest execution can continue.
12174 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12175 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12176 * interpreter.
12177 */
12178HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12179{
12180 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12181 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12182
12183 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12184 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12185 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12186 AssertRCReturn(rc, rc);
12187
12188 VBOXSTRICTRC rcStrict;
12189 PVM pVM = pVCpu->CTX_SUFF(pVM);
12190 RTGCUINTPTR const uExitQual = pVmxTransient->uExitQual;
12191 uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
12192 switch (uAccessType)
12193 {
12194 case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
12195 {
12196 uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
12197 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
12198 VMX_EXIT_QUAL_CRX_GENREG(uExitQual));
12199 AssertMsg( rcStrict == VINF_SUCCESS
12200 || rcStrict == VINF_IEM_RAISED_XCPT
12201 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12202
12203 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
12204 {
12205 case 0:
12206 {
12207 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12208 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12209 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
12210 Log4Func(("CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
12211
12212 /*
12213 * This is a kludge for handling switches back to real mode when we try to use
12214 * V86 mode to run real mode code directly. Problem is that V86 mode cannot
12215 * deal with special selector values, so we have to return to ring-3 and run
12216 * there till the selector values are V86 mode compatible.
12217 *
12218 * Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
12219 * latter is an alias for VINF_IEM_RAISED_XCPT which is converted to VINF_SUCCESs
12220 * at the end of this function.
12221 */
12222 if ( rc == VINF_SUCCESS
12223 && !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
12224 && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
12225 && (uOldCr0 & X86_CR0_PE)
12226 && !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
12227 {
12228 /** @todo check selectors rather than returning all the time. */
12229 Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
12230 rcStrict = VINF_EM_RESCHEDULE_REM;
12231 }
12232 break;
12233 }
12234
12235 case 2:
12236 {
12237 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
12238 /* Nothing to do here, CR2 it's not part of the VMCS. */
12239 break;
12240 }
12241
12242 case 3:
12243 {
12244 Assert( !pVM->hm.s.fNestedPaging
12245 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
12246 || pVCpu->hm.s.fUsingDebugLoop);
12247 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
12248 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12249 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR3);
12250 Log4Func(("CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
12251 break;
12252 }
12253
12254 case 4:
12255 {
12256 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
12257 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12258 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR4);
12259 Log4Func(("CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
12260 pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12261 break;
12262 }
12263
12264 case 8:
12265 {
12266 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
12267 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
12268 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12269 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_APIC_TPR);
12270 break;
12271 }
12272 default:
12273 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual)));
12274 break;
12275 }
12276 break;
12277 }
12278
12279 case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
12280 {
12281 Assert( !pVM->hm.s.fNestedPaging
12282 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
12283 || pVCpu->hm.s.fUsingDebugLoop
12284 || VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 3);
12285 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12286 Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 8
12287 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
12288
12289 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_GENREG(uExitQual),
12290 VMX_EXIT_QUAL_CRX_REGISTER(uExitQual));
12291 AssertMsg( rcStrict == VINF_SUCCESS
12292 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12293#ifdef VBOX_WITH_STATISTICS
12294 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
12295 {
12296 case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
12297 case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
12298 case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
12299 case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
12300 case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
12301 }
12302#endif
12303 Log4Func(("CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
12304 VBOXSTRICTRC_VAL(rcStrict)));
12305 if (VMX_EXIT_QUAL_CRX_GENREG(uExitQual) == X86_GREG_xSP)
12306 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RSP);
12307 else
12308 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12309 break;
12310 }
12311
12312 case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12313 {
12314 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12315 AssertMsg( rcStrict == VINF_SUCCESS
12316 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12317
12318 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12319 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12320 Log4Func(("CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12321 break;
12322 }
12323
12324 case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12325 {
12326 /* Note! LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here. */
12327 rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
12328 AssertRCReturn(rc, rc);
12329 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual),
12330 pVmxTransient->uGuestLinearAddr);
12331 AssertMsg( rcStrict == VINF_SUCCESS
12332 || rcStrict == VINF_IEM_RAISED_XCPT
12333 , ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12334
12335 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12336 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12337 Log4Func(("LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12338 break;
12339 }
12340
12341 default:
12342 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12343 VERR_VMX_UNEXPECTED_EXCEPTION);
12344 }
12345
12346 Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS))
12347 == (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS));
12348 if (rcStrict == VINF_IEM_RAISED_XCPT)
12349 {
12350 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
12351 rcStrict = VINF_SUCCESS;
12352 }
12353
12354 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12355 NOREF(pVM);
12356 return rcStrict;
12357}
12358
12359
12360/**
12361 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12362 * VM-exit.
12363 */
12364HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12365{
12366 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12367 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12368
12369 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12370 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12371 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12372 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_EFER);
12373 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12374 AssertRCReturn(rc, rc);
12375
12376 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12377 uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
12378 uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQual);
12379 bool fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12380 bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
12381 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
12382 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12383 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12384
12385 /*
12386 * Update exit history to see if this exit can be optimized.
12387 */
12388 VBOXSTRICTRC rcStrict;
12389 PCEMEXITREC pExitRec = NULL;
12390 if ( !fGstStepping
12391 && !fDbgStepping)
12392 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12393 !fIOString
12394 ? !fIOWrite
12395 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12396 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12397 : !fIOWrite
12398 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12399 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12400 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12401 if (!pExitRec)
12402 {
12403 /* I/O operation lookup arrays. */
12404 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12405 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12406 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12407 uint32_t const cbInstr = pVmxTransient->cbInstr;
12408 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12409 PVM pVM = pVCpu->CTX_SUFF(pVM);
12410 if (fIOString)
12411 {
12412 /*
12413 * INS/OUTS - I/O String instruction.
12414 *
12415 * Use instruction-information if available, otherwise fall back on
12416 * interpreting the instruction.
12417 */
12418 Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12419 AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
12420 bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
12421 if (fInsOutsInfo)
12422 {
12423 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12424 AssertRCReturn(rc2, rc2);
12425 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12426 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12427 IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12428 bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
12429 if (fIOWrite)
12430 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12431 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12432 else
12433 {
12434 /*
12435 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12436 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12437 * See Intel Instruction spec. for "INS".
12438 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12439 */
12440 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12441 }
12442 }
12443 else
12444 rcStrict = IEMExecOne(pVCpu);
12445
12446 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12447 fUpdateRipAlready = true;
12448 }
12449 else
12450 {
12451 /*
12452 * IN/OUT - I/O instruction.
12453 */
12454 Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12455 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12456 Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
12457 if (fIOWrite)
12458 {
12459 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
12460 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12461 if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
12462 && !pCtx->eflags.Bits.u1TF)
12463 rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
12464 }
12465 else
12466 {
12467 uint32_t u32Result = 0;
12468 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12469 if (IOM_SUCCESS(rcStrict))
12470 {
12471 /* Save result of I/O IN instr. in AL/AX/EAX. */
12472 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12473 }
12474 if ( rcStrict == VINF_IOM_R3_IOPORT_READ
12475 && !pCtx->eflags.Bits.u1TF)
12476 rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
12477 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12478 }
12479 }
12480
12481 if (IOM_SUCCESS(rcStrict))
12482 {
12483 if (!fUpdateRipAlready)
12484 {
12485 hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
12486 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12487 }
12488
12489 /*
12490 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12491 * while booting Fedora 17 64-bit guest.
12492 *
12493 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12494 */
12495 if (fIOString)
12496 {
12497 /** @todo Single-step for INS/OUTS with REP prefix? */
12498 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12499 }
12500 else if ( !fDbgStepping
12501 && fGstStepping)
12502 {
12503 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12504 AssertRCReturn(rc, rc);
12505 }
12506
12507 /*
12508 * If any I/O breakpoints are armed, we need to check if one triggered
12509 * and take appropriate action.
12510 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12511 */
12512 rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_DR7);
12513 AssertRCReturn(rc, rc);
12514
12515 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12516 * execution engines about whether hyper BPs and such are pending. */
12517 uint32_t const uDr7 = pCtx->dr[7];
12518 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12519 && X86_DR7_ANY_RW_IO(uDr7)
12520 && (pCtx->cr4 & X86_CR4_DE))
12521 || DBGFBpIsHwIoArmed(pVM)))
12522 {
12523 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12524
12525 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12526 VMMRZCallRing3Disable(pVCpu);
12527 HM_DISABLE_PREEMPT(pVCpu);
12528
12529 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12530
12531 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
12532 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12533 {
12534 /* Raise #DB. */
12535 if (fIsGuestDbgActive)
12536 ASMSetDR6(pCtx->dr[6]);
12537 if (pCtx->dr[7] != uDr7)
12538 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR7;
12539
12540 hmR0VmxSetPendingXcptDB(pVCpu);
12541 }
12542 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12543 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12544 else if ( rcStrict2 != VINF_SUCCESS
12545 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12546 rcStrict = rcStrict2;
12547 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12548
12549 HM_RESTORE_PREEMPT();
12550 VMMRZCallRing3Enable(pVCpu);
12551 }
12552 }
12553
12554#ifdef VBOX_STRICT
12555 if ( rcStrict == VINF_IOM_R3_IOPORT_READ
12556 || rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
12557 Assert(!fIOWrite);
12558 else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
12559 || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
12560 || rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
12561 Assert(fIOWrite);
12562 else
12563 {
12564# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12565 * statuses, that the VMM device and some others may return. See
12566 * IOM_SUCCESS() for guidance. */
12567 AssertMsg( RT_FAILURE(rcStrict)
12568 || rcStrict == VINF_SUCCESS
12569 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12570 || rcStrict == VINF_EM_DBG_BREAKPOINT
12571 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12572 || rcStrict == VINF_EM_RAW_TO_R3
12573 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12574# endif
12575 }
12576#endif
12577 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12578 }
12579 else
12580 {
12581 /*
12582 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12583 */
12584 int rc2 = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12585 AssertRCReturn(rc2, rc2);
12586 STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12587 : fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12588 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12589 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12590 VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
12591 fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12592
12593 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12594 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12595
12596 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12597 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12598 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12599 }
12600 return rcStrict;
12601}
12602
12603
12604/**
12605 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12606 * VM-exit.
12607 */
12608HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12609{
12610 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12611
12612 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12613 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12614 AssertRCReturn(rc, rc);
12615 if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12616 {
12617 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12618 AssertRCReturn(rc, rc);
12619 if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
12620 {
12621 uint32_t uErrCode;
12622 RTGCUINTPTR GCPtrFaultAddress;
12623 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12624 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12625 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
12626 if (fErrorCodeValid)
12627 {
12628 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12629 AssertRCReturn(rc, rc);
12630 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12631 }
12632 else
12633 uErrCode = 0;
12634
12635 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12636 && uVector == X86_XCPT_PF)
12637 GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
12638 else
12639 GCPtrFaultAddress = 0;
12640
12641 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12642 AssertRCReturn(rc, rc);
12643
12644 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12645 pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
12646
12647 Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
12648 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12649 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12650 }
12651 }
12652
12653 /* Fall back to the interpreter to emulate the task-switch. */
12654 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12655 return VERR_EM_INTERPRETER;
12656}
12657
12658
12659/**
12660 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12661 */
12662HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12663{
12664 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12665 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_MONITOR_TRAP_FLAG);
12666 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
12667 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12668 AssertRCReturn(rc, rc);
12669 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12670 return VINF_EM_DBG_STEPPED;
12671}
12672
12673
12674/**
12675 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12676 */
12677HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12678{
12679 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12680
12681 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12682
12683 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12684 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12685 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12686 {
12687 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12688 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12689 {
12690 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12691 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12692 }
12693 }
12694 else
12695 {
12696 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12697 rcStrict1 = VINF_SUCCESS;
12698 return rcStrict1;
12699 }
12700
12701 /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12702 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12703 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12704 AssertRCReturn(rc, rc);
12705
12706 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12707 uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
12708 VBOXSTRICTRC rcStrict2;
12709 switch (uAccessType)
12710 {
12711 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12712 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12713 {
12714 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
12715 || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
12716 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12717
12718 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12719 GCPhys &= PAGE_BASE_GC_MASK;
12720 GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
12721 PVM pVM = pVCpu->CTX_SUFF(pVM);
12722 Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12723 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
12724
12725 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12726 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12727 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12728 CPUMCTX2CORE(pCtx), GCPhys);
12729 Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12730 if ( rcStrict2 == VINF_SUCCESS
12731 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12732 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12733 {
12734 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12735 | HM_CHANGED_GUEST_APIC_TPR);
12736 rcStrict2 = VINF_SUCCESS;
12737 }
12738 break;
12739 }
12740
12741 default:
12742 Log4Func(("uAccessType=%#x\n", uAccessType));
12743 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12744 break;
12745 }
12746
12747 if (rcStrict2 != VINF_SUCCESS)
12748 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12749 return rcStrict2;
12750}
12751
12752
12753/**
12754 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12755 * VM-exit.
12756 */
12757HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12758{
12759 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12760
12761 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12762 if (pVmxTransient->fWasGuestDebugStateActive)
12763 {
12764 AssertMsgFailed(("Unexpected MOV DRx exit\n"));
12765 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12766 }
12767
12768 if ( !pVCpu->hm.s.fSingleInstruction
12769 && !pVmxTransient->fWasHyperDebugStateActive)
12770 {
12771 Assert(!DBGFIsStepping(pVCpu));
12772 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12773
12774 /* Don't intercept MOV DRx any more. */
12775 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
12776 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12777 AssertRCReturn(rc, rc);
12778
12779 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12780 VMMRZCallRing3Disable(pVCpu);
12781 HM_DISABLE_PREEMPT(pVCpu);
12782
12783 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12784 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12785 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12786
12787 HM_RESTORE_PREEMPT();
12788 VMMRZCallRing3Enable(pVCpu);
12789
12790#ifdef VBOX_WITH_STATISTICS
12791 rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12792 AssertRCReturn(rc, rc);
12793 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12794 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12795 else
12796 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12797#endif
12798 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12799 return VINF_SUCCESS;
12800 }
12801
12802 /*
12803 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12804 * Update the segment registers and DR7 from the CPU.
12805 */
12806 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12807 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12808 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_DR7);
12809 AssertRCReturn(rc, rc);
12810 Log4Func(("CS:RIP=%04x:%08RX64\n", pCtx->cs.Sel, pCtx->rip));
12811
12812 PVM pVM = pVCpu->CTX_SUFF(pVM);
12813 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12814 {
12815 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12816 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
12817 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
12818 if (RT_SUCCESS(rc))
12819 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12820 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12821 }
12822 else
12823 {
12824 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12825 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
12826 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
12827 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12828 }
12829
12830 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12831 if (RT_SUCCESS(rc))
12832 {
12833 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12834 AssertRCReturn(rc2, rc2);
12835 return VINF_SUCCESS;
12836 }
12837 return rc;
12838}
12839
12840
12841/**
12842 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12843 * Conditional VM-exit.
12844 */
12845HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12846{
12847 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12848 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12849
12850 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12851 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12852 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12853 {
12854 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12855 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12856 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12857 {
12858 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12859 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12860 }
12861 }
12862 else
12863 {
12864 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12865 rcStrict1 = VINF_SUCCESS;
12866 return rcStrict1;
12867 }
12868
12869 /*
12870 * Get sufficent state and update the exit history entry.
12871 */
12872 RTGCPHYS GCPhys;
12873 int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
12874 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12875 AssertRCReturn(rc, rc);
12876
12877 VBOXSTRICTRC rcStrict;
12878 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12879 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
12880 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12881 if (!pExitRec)
12882 {
12883 /*
12884 * If we succeed, resume guest execution.
12885 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12886 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12887 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12888 * weird case. See @bugref{6043}.
12889 */
12890 PVM pVM = pVCpu->CTX_SUFF(pVM);
12891 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12892 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
12893 Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12894 if ( rcStrict == VINF_SUCCESS
12895 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12896 || rcStrict == VERR_PAGE_NOT_PRESENT)
12897 {
12898 /* Successfully handled MMIO operation. */
12899 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12900 | HM_CHANGED_GUEST_APIC_TPR);
12901 rcStrict = VINF_SUCCESS;
12902 }
12903 }
12904 else
12905 {
12906 /*
12907 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12908 */
12909 int rc2 = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12910 AssertRCReturn(rc2, rc2);
12911
12912 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12913 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12914
12915 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12916 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12917
12918 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12919 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12920 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12921 }
12922 return VBOXSTRICTRC_TODO(rcStrict);
12923}
12924
12925
12926/**
12927 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12928 * VM-exit.
12929 */
12930HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12931{
12932 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12933 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12934
12935 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12936 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12937 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12938 {
12939 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12940 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12941 Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12942 }
12943 else
12944 {
12945 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12946 rcStrict1 = VINF_SUCCESS;
12947 return rcStrict1;
12948 }
12949
12950 RTGCPHYS GCPhys;
12951 int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
12952 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12953 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12954 AssertRCReturn(rc, rc);
12955
12956 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12957 AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
12958
12959 RTGCUINT uErrorCode = 0;
12960 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12961 uErrorCode |= X86_TRAP_PF_ID;
12962 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12963 uErrorCode |= X86_TRAP_PF_RW;
12964 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12965 uErrorCode |= X86_TRAP_PF_P;
12966
12967 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12968
12969
12970 /* Handle the pagefault trap for the nested shadow table. */
12971 PVM pVM = pVCpu->CTX_SUFF(pVM);
12972 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12973
12974 Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
12975 pCtx->cs.Sel, pCtx->rip));
12976
12977 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
12978 TRPMResetTrap(pVCpu);
12979
12980 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12981 if ( rcStrict2 == VINF_SUCCESS
12982 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12983 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12984 {
12985 /* Successfully synced our nested page tables. */
12986 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12987 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
12988 return VINF_SUCCESS;
12989 }
12990
12991 Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12992 return rcStrict2;
12993}
12994
12995/** @} */
12996
12997/** @name VM-exit exception handlers.
12998 * @{
12999 */
13000/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13001/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
13002/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13003
13004/**
13005 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
13006 */
13007static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13008{
13009 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13010 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13011
13012 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0);
13013 AssertRCReturn(rc, rc);
13014
13015 if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
13016 {
13017 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13018 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13019
13020 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13021 * provides VM-exit instruction length. If this causes problem later,
13022 * disassemble the instruction like it's done on AMD-V. */
13023 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13024 AssertRCReturn(rc2, rc2);
13025 return rc;
13026 }
13027
13028 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13029 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13030 return rc;
13031}
13032
13033
13034/**
13035 * VM-exit exception handler for \#BP (Breakpoint exception).
13036 */
13037static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13038{
13039 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13040 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13041
13042 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13043 AssertRCReturn(rc, rc);
13044
13045 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13046 rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13047 if (rc == VINF_EM_RAW_GUEST_TRAP)
13048 {
13049 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13050 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13051 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13052 AssertRCReturn(rc, rc);
13053
13054 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13055 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13056 }
13057
13058 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
13059 return rc;
13060}
13061
13062
13063/**
13064 * VM-exit exception handler for \#AC (alignment check exception).
13065 */
13066static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13067{
13068 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13069
13070 /*
13071 * Re-inject it. We'll detect any nesting before getting here.
13072 */
13073 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13074 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13075 AssertRCReturn(rc, rc);
13076 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13077
13078 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13079 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13080 return VINF_SUCCESS;
13081}
13082
13083
13084/**
13085 * VM-exit exception handler for \#DB (Debug exception).
13086 */
13087static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13088{
13089 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13090 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13091
13092 /*
13093 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13094 * for processing.
13095 */
13096 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13097
13098 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13099 uint64_t uDR6 = X86_DR6_INIT_VAL;
13100 uDR6 |= (pVmxTransient->uExitQual & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
13101
13102 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13103 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13104 Log6Func(("rc=%Rrc\n", rc));
13105 if (rc == VINF_EM_RAW_GUEST_TRAP)
13106 {
13107 /*
13108 * The exception was for the guest. Update DR6, DR7.GD and
13109 * IA32_DEBUGCTL.LBR before forwarding it.
13110 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13111 */
13112 VMMRZCallRing3Disable(pVCpu);
13113 HM_DISABLE_PREEMPT(pVCpu);
13114
13115 pCtx->dr[6] &= ~X86_DR6_B_MASK;
13116 pCtx->dr[6] |= uDR6;
13117 if (CPUMIsGuestDebugStateActive(pVCpu))
13118 ASMSetDR6(pCtx->dr[6]);
13119
13120 HM_RESTORE_PREEMPT();
13121 VMMRZCallRing3Enable(pVCpu);
13122
13123 rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_DR7);
13124 AssertRCReturn(rc, rc);
13125
13126 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13127 pCtx->dr[7] &= ~X86_DR7_GD;
13128
13129 /* Paranoia. */
13130 pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13131 pCtx->dr[7] |= X86_DR7_RA1_MASK;
13132
13133 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
13134 AssertRCReturn(rc, rc);
13135
13136 /*
13137 * Raise #DB in the guest.
13138 *
13139 * It is important to reflect exactly what the VM-exit gave us (preserving the
13140 * interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
13141 * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
13142 * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
13143 *
13144 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
13145 * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
13146 */
13147 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13148 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13149 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13150 AssertRCReturn(rc, rc);
13151 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13152 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13153 return VINF_SUCCESS;
13154 }
13155
13156 /*
13157 * Not a guest trap, must be a hypervisor related debug event then.
13158 * Update DR6 in case someone is interested in it.
13159 */
13160 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13161 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13162 CPUMSetHyperDR6(pVCpu, uDR6);
13163
13164 return rc;
13165}
13166
13167
13168/**
13169 * Hacks its way around the lovely mesa driver's backdoor accesses.
13170 */
13171static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
13172{
13173 Log(("hmR0VmxHandleMesaDrvGp: at %04x:%08RX64 rcx=%RX64 rbx=%RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
13174 RT_NOREF(pCtx);
13175
13176 /* For now we'll just skip the instruction. */
13177 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13178}
13179
13180
13181/**
13182 * Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
13183 * backdoor logging w/o checking what it is running inside.
13184 *
13185 * This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
13186 * backdoor port and magic numbers loaded in registers.
13187 *
13188 * @returns true if it is, false if it isn't.
13189 */
13190DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
13191{
13192 /* 0xed: IN eAX,dx */
13193 uint8_t abInstr[1];
13194 if (pVmxTransient->cbInstr != sizeof(abInstr))
13195 return false;
13196
13197 /* Check that it is #GP(0). */
13198 if (pVmxTransient->uExitIntErrorCode != 0)
13199 return false;
13200
13201 /* Check magic and port. */
13202 Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_RCX)));
13203 /*Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));*/
13204 if (pCtx->rax != UINT32_C(0x564d5868))
13205 return false;
13206 if (pCtx->dx != UINT32_C(0x5658))
13207 return false;
13208
13209 /* Flat ring-3 CS. */
13210 AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
13211 Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
13212 /*Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));*/
13213 if (pCtx->cs.Attr.n.u2Dpl != 3)
13214 return false;
13215 if (pCtx->cs.u64Base != 0)
13216 return false;
13217
13218 /* Check opcode. */
13219 AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
13220 Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
13221 int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
13222 /*Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));*/
13223 if (RT_FAILURE(rc))
13224 return false;
13225 if (abInstr[0] != 0xed)
13226 return false;
13227
13228 return true;
13229}
13230
13231
13232/**
13233 * VM-exit exception handler for \#GP (General-protection exception).
13234 *
13235 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13236 */
13237static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13238{
13239 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13240 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13241
13242 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13243 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13244 { /* likely */ }
13245 else
13246 {
13247#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13248 Assert(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv);
13249#endif
13250 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13251 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13252 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13253 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13254 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13255 AssertRCReturn(rc, rc);
13256 Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pCtx->cs.Sel, pCtx->rip,
13257 pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
13258
13259 if ( !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
13260 || !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
13261 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13262 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13263 else
13264 rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
13265 return rc;
13266 }
13267
13268 Assert(CPUMIsGuestInRealModeEx(pCtx));
13269 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13270
13271 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13272 AssertRCReturn(rc, rc);
13273
13274 VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
13275 if (rcStrict == VINF_SUCCESS)
13276 {
13277 if (!CPUMIsGuestInRealModeEx(pCtx))
13278 {
13279 /*
13280 * The guest is no longer in real-mode, check if we can continue executing the
13281 * guest using hardware-assisted VMX. Otherwise, fall back to emulation.
13282 */
13283 if (HMVmxCanExecuteGuest(pVCpu, pCtx))
13284 {
13285 Log4Func(("Mode changed but guest still suitable for executing using VT-x\n"));
13286 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
13287 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13288 }
13289 else
13290 {
13291 Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
13292 rcStrict = VINF_EM_RESCHEDULE;
13293 }
13294 }
13295 else
13296 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13297 }
13298 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13299 {
13300 rcStrict = VINF_SUCCESS;
13301 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13302 }
13303 return VBOXSTRICTRC_VAL(rcStrict);
13304}
13305
13306
13307/**
13308 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13309 * the exception reported in the VMX transient structure back into the VM.
13310 *
13311 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13312 * up-to-date.
13313 */
13314static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13315{
13316 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13317#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13318 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13319 ("uVector=%#x u32XcptBitmap=%#X32\n",
13320 VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13321#endif
13322
13323 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13324 hmR0VmxCheckExitDueToEventDelivery(). */
13325 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13326 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13327 AssertRCReturn(rc, rc);
13328 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13329
13330#ifdef DEBUG_ramshankar
13331 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
13332 uint8_t uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13333 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13334#endif
13335
13336 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13337 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13338 return VINF_SUCCESS;
13339}
13340
13341
13342/**
13343 * VM-exit exception handler for \#PF (Page-fault exception).
13344 */
13345static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13346{
13347 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13348 PVM pVM = pVCpu->CTX_SUFF(pVM);
13349 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13350 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13351 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13352 AssertRCReturn(rc, rc);
13353
13354 if (!pVM->hm.s.fNestedPaging)
13355 { /* likely */ }
13356 else
13357 {
13358#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13359 Assert(pVCpu->hm.s.fUsingDebugLoop);
13360#endif
13361 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13362 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13363 {
13364 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13365 pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
13366 }
13367 else
13368 {
13369 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13370 hmR0VmxSetPendingXcptDF(pVCpu);
13371 Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13372 }
13373 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13374 return rc;
13375 }
13376
13377 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13378 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13379 if (pVmxTransient->fVectoringPF)
13380 {
13381 Assert(pVCpu->hm.s.Event.fPending);
13382 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13383 }
13384
13385 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13386 rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13387 AssertRCReturn(rc, rc);
13388
13389 Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
13390 pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
13391
13392 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13393 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
13394
13395 Log4Func(("#PF: rc=%Rrc\n", rc));
13396 if (rc == VINF_SUCCESS)
13397 {
13398 /*
13399 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13400 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13401 */
13402 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13403 TRPMResetTrap(pVCpu);
13404 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13405 return rc;
13406 }
13407
13408 if (rc == VINF_EM_RAW_GUEST_TRAP)
13409 {
13410 if (!pVmxTransient->fVectoringDoublePF)
13411 {
13412 /* It's a guest page fault and needs to be reflected to the guest. */
13413 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13414 TRPMResetTrap(pVCpu);
13415 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13416 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13417 uGstErrorCode, pVmxTransient->uExitQual);
13418 }
13419 else
13420 {
13421 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13422 TRPMResetTrap(pVCpu);
13423 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13424 hmR0VmxSetPendingXcptDF(pVCpu);
13425 Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13426 }
13427
13428 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13429 return VINF_SUCCESS;
13430 }
13431
13432 TRPMResetTrap(pVCpu);
13433 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13434 return rc;
13435}
13436
13437/** @} */
13438
13439#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13440/** @name Nested-guest VM-exit handlers.
13441 * @{
13442 */
13443/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13444/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= Nested-guest VM-exit handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13445/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13446
13447/**
13448 * VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
13449 */
13450HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13451{
13452 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13453
13454 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13455 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13456 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13457 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13458 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13459 AssertRCReturn(rc, rc);
13460
13461 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13462
13463 VMXVEXITINFO ExitInfo;
13464 RT_ZERO(ExitInfo);
13465 ExitInfo.uReason = pVmxTransient->uExitReason;
13466 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13467 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13468 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13469 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13470
13471 VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
13472 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13473 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13474 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13475 {
13476 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13477 rcStrict = VINF_SUCCESS;
13478 }
13479 return rcStrict;
13480}
13481
13482
13483/**
13484 * VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
13485 */
13486HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13487{
13488 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13489
13490 /** @todo NSTVMX: Vmlaunch. */
13491 hmR0VmxSetPendingXcptUD(pVCpu);
13492 return VINF_SUCCESS;
13493}
13494
13495
13496/**
13497 * VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
13498 */
13499HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13500{
13501 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13502
13503 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13504 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13505 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13506 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13507 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13508 AssertRCReturn(rc, rc);
13509
13510 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13511
13512 VMXVEXITINFO ExitInfo;
13513 RT_ZERO(ExitInfo);
13514 ExitInfo.uReason = pVmxTransient->uExitReason;
13515 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13516 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13517 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13518 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13519
13520 VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
13521 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13522 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13523 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13524 {
13525 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13526 rcStrict = VINF_SUCCESS;
13527 }
13528 return rcStrict;
13529}
13530
13531
13532/**
13533 * VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
13534 */
13535HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13536{
13537 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13538
13539 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13540 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13541 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13542 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13543 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13544 AssertRCReturn(rc, rc);
13545
13546 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13547
13548 VMXVEXITINFO ExitInfo;
13549 RT_ZERO(ExitInfo);
13550 ExitInfo.uReason = pVmxTransient->uExitReason;
13551 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13552 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13553 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13554 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
13555
13556 VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
13557 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13558 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13559 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13560 {
13561 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13562 rcStrict = VINF_SUCCESS;
13563 }
13564 return rcStrict;
13565}
13566
13567
13568/**
13569 * VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Unconditional VM-exit.
13570 */
13571HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13572{
13573 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13574
13575 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13576 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13577 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13578 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13579 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13580 AssertRCReturn(rc, rc);
13581
13582 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13583
13584 VMXVEXITINFO ExitInfo;
13585 RT_ZERO(ExitInfo);
13586 ExitInfo.uReason = pVmxTransient->uExitReason;
13587 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13588 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13589 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13590 if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
13591 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
13592
13593 VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
13594 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13595 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13596 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13597 {
13598 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13599 rcStrict = VINF_SUCCESS;
13600 }
13601 return rcStrict;
13602}
13603
13604
13605/**
13606 * VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
13607 */
13608HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13609{
13610 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13611
13612 /** @todo NSTVMX: Vmresume. */
13613 hmR0VmxSetPendingXcptUD(pVCpu);
13614 return VINF_SUCCESS;
13615}
13616
13617
13618/**
13619 * VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Unconditional VM-exit.
13620 */
13621HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13622{
13623 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13624
13625 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13626 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13627 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13628 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13629 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13630 AssertRCReturn(rc, rc);
13631
13632 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13633
13634 VMXVEXITINFO ExitInfo;
13635 RT_ZERO(ExitInfo);
13636 ExitInfo.uReason = pVmxTransient->uExitReason;
13637 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13638 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13639 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13640 if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
13641 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13642
13643 VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
13644 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13645 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13646 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13647 {
13648 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13649 rcStrict = VINF_SUCCESS;
13650 }
13651 return rcStrict;
13652}
13653
13654
13655/**
13656 * VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
13657 */
13658HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13659{
13660 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13661
13662 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13663 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13664 AssertRCReturn(rc, rc);
13665
13666 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13667
13668 VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
13669 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13670 {
13671 /* VMXOFF on success changes the internal hwvirt state but not anything that's visible to the guest. */
13672 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
13673 }
13674 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13675 {
13676 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13677 rcStrict = VINF_SUCCESS;
13678 }
13679 return rcStrict;
13680}
13681
13682
13683/**
13684 * VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
13685 */
13686HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13687{
13688 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13689
13690 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13691 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13692 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13693 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13694 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13695 AssertRCReturn(rc, rc);
13696
13697 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13698
13699 VMXVEXITINFO ExitInfo;
13700 RT_ZERO(ExitInfo);
13701 ExitInfo.uReason = pVmxTransient->uExitReason;
13702 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13703 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13704 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13705 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13706
13707 VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
13708 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13709 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13710 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13711 {
13712 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13713 rcStrict = VINF_SUCCESS;
13714 }
13715 return rcStrict;
13716}
13717
13718/** @} */
13719#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13720
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette