VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 75265

Last change on this file since 75265 was 75265, checked in by vboxsync, 6 years ago

HMVMXR0.cpp: Fixed lazy state import issue in debug event code.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 575.9 KB
Line 
1/* $Id: HMVMXR0.cpp 75265 2018-11-06 03:13:52Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/x86.h>
25#include <iprt/asm-amd64-x86.h>
26#include <iprt/thread.h>
27
28#include <VBox/vmm/pdmapi.h>
29#include <VBox/vmm/dbgf.h>
30#include <VBox/vmm/iem.h>
31#include <VBox/vmm/iom.h>
32#include <VBox/vmm/selm.h>
33#include <VBox/vmm/tm.h>
34#include <VBox/vmm/em.h>
35#include <VBox/vmm/gim.h>
36#include <VBox/vmm/apic.h>
37#ifdef VBOX_WITH_REM
38# include <VBox/vmm/rem.h>
39#endif
40#include "HMInternal.h"
41#include <VBox/vmm/vm.h>
42#include "HMVMXR0.h"
43#include "dtrace/VBoxVMM.h"
44
45#ifdef DEBUG_ramshankar
46# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
47# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
49# define HMVMX_ALWAYS_CHECK_GUEST_STATE
50# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
51# define HMVMX_ALWAYS_TRAP_PF
52# define HMVMX_ALWAYS_FLUSH_TLB
53# define HMVMX_ALWAYS_SWAP_EFER
54#endif
55
56
57/*********************************************************************************************************************************
58* Defined Constants And Macros *
59*********************************************************************************************************************************/
60/** Use the function table. */
61#define HMVMX_USE_FUNCTION_TABLE
62
63/** Determine which tagged-TLB flush handler to use. */
64#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
65#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
66#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
67#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
68
69/** @name HMVMX_READ_XXX
70 * Flags to skip redundant reads of some common VMCS fields that are not part of
71 * the guest-CPU or VCPU state but are needed while handling VM-exits.
72 */
73#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
74#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
75#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
76#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
77#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
78#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
79#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
80#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
81/** @} */
82
83/**
84 * States of the VMCS.
85 *
86 * This does not reflect all possible VMCS states but currently only those
87 * needed for maintaining the VMCS consistently even when thread-context hooks
88 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
89 */
90#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
91#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
92#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
93
94/**
95 * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
96 * guest using hardware-assisted VMX.
97 *
98 * This excludes state like GPRs (other than RSP) which are always are
99 * swapped and restored across the world-switch and also registers like EFER,
100 * MSR which cannot be modified by the guest without causing a VM-exit.
101 */
102#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
103 | CPUMCTX_EXTRN_RFLAGS \
104 | CPUMCTX_EXTRN_RSP \
105 | CPUMCTX_EXTRN_SREG_MASK \
106 | CPUMCTX_EXTRN_TABLE_MASK \
107 | CPUMCTX_EXTRN_KERNEL_GS_BASE \
108 | CPUMCTX_EXTRN_SYSCALL_MSRS \
109 | CPUMCTX_EXTRN_SYSENTER_MSRS \
110 | CPUMCTX_EXTRN_TSC_AUX \
111 | CPUMCTX_EXTRN_OTHER_MSRS \
112 | CPUMCTX_EXTRN_CR0 \
113 | CPUMCTX_EXTRN_CR3 \
114 | CPUMCTX_EXTRN_CR4 \
115 | CPUMCTX_EXTRN_DR7 \
116 | CPUMCTX_EXTRN_HM_VMX_MASK)
117
118/**
119 * Exception bitmap mask for real-mode guests (real-on-v86).
120 *
121 * We need to intercept all exceptions manually except:
122 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
123 * due to bugs in Intel CPUs.
124 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
125 * support.
126 */
127#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
128 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
129 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
130 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
131 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
132 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
133 | RT_BIT(X86_XCPT_XF))
134
135/** Maximum VM-instruction error number. */
136#define HMVMX_INSTR_ERROR_MAX 28
137
138/** Profiling macro. */
139#ifdef HM_PROFILE_EXIT_DISPATCH
140# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
141# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
142#else
143# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
144# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
145#endif
146
147/** Assert that preemption is disabled or covered by thread-context hooks. */
148#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
149 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
150
151/** Assert that we haven't migrated CPUs when thread-context hooks are not
152 * used. */
153#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
154 || (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
155 ("Illegal migration! Entered on CPU %u Current %u\n", \
156 (a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
157
158/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
159 * context. */
160#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
161 ("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
162 (a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
163
164/** Macro for importing guest state from the VMCS back into CPUMCTX (intended to be
165 * used only from VM-exit handlers). */
166#define HMVMX_CPUMCTX_IMPORT_STATE(a_pVCpu, a_fWhat) (hmR0VmxImportGuestState((a_pVCpu), (a_fWhat)))
167
168/** Helper macro for VM-exit handlers called unexpectedly. */
169#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
170 do { \
171 (a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
172 return VERR_VMX_UNEXPECTED_EXIT; \
173 } while (0)
174
175/** Macro for importing segment registers to the VMCS from the guest-CPU context. */
176#ifdef VMX_USE_CACHED_VMCS_ACCESSES
177# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
178 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
179 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
180#else
181# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
182 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
183 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
184#endif
185
186/** Macro for exporting segment registers to the VMCS from the guest-CPU context. */
187#define HMVMX_EXPORT_SREG(Sel, a_pCtxSelReg) \
188 hmR0VmxExportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
189 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
190
191#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
192/** Macro that does the necessary privilege checks and intercepted VM-exits for
193 * guests that attempted to execute a VMX instruction. */
194# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
195 do \
196 { \
197 VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
198 if (rcStrictTmp == VINF_SUCCESS) \
199 { /* likely */ } \
200 else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
201 { \
202 Assert((a_pVCpu)->hm.s.Event.fPending); \
203 Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
204 return VINF_SUCCESS; \
205 } \
206 else \
207 { \
208 int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
209 AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
210 } \
211 } while (0)
212
213/** Macro that decodes a memory operand for an instruction VM-exit. */
214# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
215 do \
216 { \
217 VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
218 (a_pGCPtrEffAddr)); \
219 if (rcStrictTmp == VINF_SUCCESS) \
220 { /* likely */ } \
221 else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
222 { \
223 uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
224 Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
225 return VINF_SUCCESS; \
226 } \
227 else \
228 { \
229 Log4Func(("hmR0VmxCheckExitDueToVmxInstr failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
230 return rcStrictTmp; \
231 } \
232 } while (0)
233
234#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
235
236
237/*********************************************************************************************************************************
238* Structures and Typedefs *
239*********************************************************************************************************************************/
240/**
241 * VMX transient state.
242 *
243 * A state structure for holding miscellaneous information across
244 * VMX non-root operation and restored after the transition.
245 */
246typedef struct VMXTRANSIENT
247{
248 /** The host's rflags/eflags. */
249 RTCCUINTREG fEFlags;
250#if HC_ARCH_BITS == 32
251 uint32_t u32Alignment0;
252#endif
253 /** The guest's TPR value used for TPR shadowing. */
254 uint8_t u8GuestTpr;
255 /** Alignment. */
256 uint8_t abAlignment0[7];
257
258 /** The basic VM-exit reason. */
259 uint16_t uExitReason;
260 /** Alignment. */
261 uint16_t u16Alignment0;
262 /** The VM-exit interruption error code. */
263 uint32_t uExitIntErrorCode;
264 /** The VM-exit exit code qualification. */
265 uint64_t uExitQual;
266 /** The Guest-linear address. */
267 uint64_t uGuestLinearAddr;
268
269 /** The VM-exit interruption-information field. */
270 uint32_t uExitIntInfo;
271 /** The VM-exit instruction-length field. */
272 uint32_t cbInstr;
273 /** The VM-exit instruction-information field. */
274 VMXEXITINSTRINFO ExitInstrInfo;
275 /** Whether the VM-entry failed or not. */
276 bool fVMEntryFailed;
277 /** Alignment. */
278 uint8_t abAlignment1[3];
279
280 /** The VM-entry interruption-information field. */
281 uint32_t uEntryIntInfo;
282 /** The VM-entry exception error code field. */
283 uint32_t uEntryXcptErrorCode;
284 /** The VM-entry instruction length field. */
285 uint32_t cbEntryInstr;
286
287 /** IDT-vectoring information field. */
288 uint32_t uIdtVectoringInfo;
289 /** IDT-vectoring error code. */
290 uint32_t uIdtVectoringErrorCode;
291
292 /** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
293 uint32_t fVmcsFieldsRead;
294
295 /** Whether the guest debug state was active at the time of VM-exit. */
296 bool fWasGuestDebugStateActive;
297 /** Whether the hyper debug state was active at the time of VM-exit. */
298 bool fWasHyperDebugStateActive;
299 /** Whether TSC-offsetting should be setup before VM-entry. */
300 bool fUpdateTscOffsettingAndPreemptTimer;
301 /** Whether the VM-exit was caused by a page-fault during delivery of a
302 * contributory exception or a page-fault. */
303 bool fVectoringDoublePF;
304 /** Whether the VM-exit was caused by a page-fault during delivery of an
305 * external interrupt or NMI. */
306 bool fVectoringPF;
307} VMXTRANSIENT;
308AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
309AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
310AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
311AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
312AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
313/** Pointer to VMX transient state. */
314typedef VMXTRANSIENT *PVMXTRANSIENT;
315
316/**
317 * Memory operand read or write access.
318 */
319typedef enum VMXMEMACCESS
320{
321 VMXMEMACCESS_READ = 0,
322 VMXMEMACCESS_WRITE = 1
323} VMXMEMACCESS;
324
325/**
326 * VMX VM-exit handler.
327 *
328 * @returns Strict VBox status code (i.e. informational status codes too).
329 * @param pVCpu The cross context virtual CPU structure.
330 * @param pVmxTransient Pointer to the VMX-transient structure.
331 */
332#ifndef HMVMX_USE_FUNCTION_TABLE
333typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
334#else
335typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
336/** Pointer to VM-exit handler. */
337typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
338#endif
339
340/**
341 * VMX VM-exit handler, non-strict status code.
342 *
343 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
344 *
345 * @returns VBox status code, no informational status code returned.
346 * @param pVCpu The cross context virtual CPU structure.
347 * @param pVmxTransient Pointer to the VMX-transient structure.
348 *
349 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
350 * use of that status code will be replaced with VINF_EM_SOMETHING
351 * later when switching over to IEM.
352 */
353#ifndef HMVMX_USE_FUNCTION_TABLE
354typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
355#else
356typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
357#endif
358
359
360/*********************************************************************************************************************************
361* Internal Functions *
362*********************************************************************************************************************************/
363static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush);
364static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr);
365static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
366static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
367static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
368 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
369#if HC_ARCH_BITS == 32
370static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
371#endif
372#ifndef HMVMX_USE_FUNCTION_TABLE
373DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
374# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
375# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
376#else
377# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
378# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
379#endif
380
381/** @name VM-exit handlers.
382 * @{
383 */
384static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
385static FNVMXEXITHANDLER hmR0VmxExitExtInt;
386static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
387static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
388static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
389static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
390static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
391static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
392static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
393static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
394static FNVMXEXITHANDLER hmR0VmxExitCpuid;
395static FNVMXEXITHANDLER hmR0VmxExitGetsec;
396static FNVMXEXITHANDLER hmR0VmxExitHlt;
397static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
398static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
399static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
400static FNVMXEXITHANDLER hmR0VmxExitVmcall;
401#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
402static FNVMXEXITHANDLER hmR0VmxExitVmclear;
403static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
404static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
405static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
406static FNVMXEXITHANDLER hmR0VmxExitVmread;
407static FNVMXEXITHANDLER hmR0VmxExitVmresume;
408static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
409static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
410static FNVMXEXITHANDLER hmR0VmxExitVmxon;
411#endif
412static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
413static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
414static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
415static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
416static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
417static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
418static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
419static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
420static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
421static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
422static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
423static FNVMXEXITHANDLER hmR0VmxExitMwait;
424static FNVMXEXITHANDLER hmR0VmxExitMtf;
425static FNVMXEXITHANDLER hmR0VmxExitMonitor;
426static FNVMXEXITHANDLER hmR0VmxExitPause;
427static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
428static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
429static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
430static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
431static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
432static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
433static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
434static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
435static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
436static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
437static FNVMXEXITHANDLER hmR0VmxExitRdrand;
438static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
439/** @} */
440
441static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
442static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
443static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
444static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
445static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
446static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
447static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
448static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu);
449
450
451/*********************************************************************************************************************************
452* Global Variables *
453*********************************************************************************************************************************/
454#ifdef HMVMX_USE_FUNCTION_TABLE
455
456/**
457 * VMX_EXIT dispatch table.
458 */
459static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
460{
461 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
462 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
463 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
464 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
465 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
466 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
467 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
468 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
469 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
470 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
471 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
472 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
473 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
474 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
475 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
476 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
477 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
478 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
479 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
480#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
481 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
482 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
483 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
484 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
485 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
486 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
487 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
488 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
489 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
490#else
491 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
492 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
493 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
494 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
495 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
496 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
497 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
498 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
499 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
500#endif
501 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
502 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
503 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
504 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
505 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
506 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
507 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
508 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
509 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
510 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
511 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
512 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
513 /* 40 UNDEFINED */ hmR0VmxExitPause,
514 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
515 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
516 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
517 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
518 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
519 /* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitXdtrAccess,
520 /* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitXdtrAccess,
521 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
522 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
523 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
524 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
525 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
526 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
527 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
528 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
529 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
530 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
531 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
532 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
533 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
534 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
535 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
536 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
537 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
538};
539#endif /* HMVMX_USE_FUNCTION_TABLE */
540
541#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
542static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
543{
544 /* 0 */ "(Not Used)",
545 /* 1 */ "VMCALL executed in VMX root operation.",
546 /* 2 */ "VMCLEAR with invalid physical address.",
547 /* 3 */ "VMCLEAR with VMXON pointer.",
548 /* 4 */ "VMLAUNCH with non-clear VMCS.",
549 /* 5 */ "VMRESUME with non-launched VMCS.",
550 /* 6 */ "VMRESUME after VMXOFF",
551 /* 7 */ "VM-entry with invalid control fields.",
552 /* 8 */ "VM-entry with invalid host state fields.",
553 /* 9 */ "VMPTRLD with invalid physical address.",
554 /* 10 */ "VMPTRLD with VMXON pointer.",
555 /* 11 */ "VMPTRLD with incorrect revision identifier.",
556 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
557 /* 13 */ "VMWRITE to read-only VMCS component.",
558 /* 14 */ "(Not Used)",
559 /* 15 */ "VMXON executed in VMX root operation.",
560 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
561 /* 17 */ "VM-entry with non-launched executing VMCS.",
562 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
563 /* 19 */ "VMCALL with non-clear VMCS.",
564 /* 20 */ "VMCALL with invalid VM-exit control fields.",
565 /* 21 */ "(Not Used)",
566 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
567 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
568 /* 24 */ "VMCALL with invalid SMM-monitor features.",
569 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
570 /* 26 */ "VM-entry with events blocked by MOV SS.",
571 /* 27 */ "(Not Used)",
572 /* 28 */ "Invalid operand to INVEPT/INVVPID."
573};
574#endif /* VBOX_STRICT */
575
576
577/**
578 * Updates the VM's last error record.
579 *
580 * If there was a VMX instruction error, reads the error data from the VMCS and
581 * updates VCPU's last error record as well.
582 *
583 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
584 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
585 * VERR_VMX_INVALID_VMCS_FIELD.
586 * @param rc The error code.
587 */
588static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
589{
590 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
591 || rc == VERR_VMX_UNABLE_TO_START_VM)
592 {
593 AssertPtrReturnVoid(pVCpu);
594 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
595 }
596 pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
597}
598
599
600/**
601 * Reads the VM-entry interruption-information field from the VMCS into the VMX
602 * transient structure.
603 *
604 * @returns VBox status code.
605 * @param pVmxTransient Pointer to the VMX transient structure.
606 *
607 * @remarks No-long-jump zone!!!
608 */
609DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
610{
611 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
612 AssertRCReturn(rc, rc);
613 return VINF_SUCCESS;
614}
615
616#ifdef VBOX_STRICT
617/**
618 * Reads the VM-entry exception error code field from the VMCS into
619 * the VMX transient structure.
620 *
621 * @returns VBox status code.
622 * @param pVmxTransient Pointer to the VMX transient structure.
623 *
624 * @remarks No-long-jump zone!!!
625 */
626DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
627{
628 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
629 AssertRCReturn(rc, rc);
630 return VINF_SUCCESS;
631}
632
633
634/**
635 * Reads the VM-entry exception error code field from the VMCS into
636 * the VMX transient structure.
637 *
638 * @returns VBox status code.
639 * @param pVmxTransient Pointer to the VMX transient structure.
640 *
641 * @remarks No-long-jump zone!!!
642 */
643DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
644{
645 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
646 AssertRCReturn(rc, rc);
647 return VINF_SUCCESS;
648}
649#endif /* VBOX_STRICT */
650
651
652/**
653 * Reads the VM-exit interruption-information field from the VMCS into the VMX
654 * transient structure.
655 *
656 * @returns VBox status code.
657 * @param pVmxTransient Pointer to the VMX transient structure.
658 */
659DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
660{
661 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
662 {
663 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
664 AssertRCReturn(rc,rc);
665 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_INFO;
666 }
667 return VINF_SUCCESS;
668}
669
670
671/**
672 * Reads the VM-exit interruption error code from the VMCS into the VMX
673 * transient structure.
674 *
675 * @returns VBox status code.
676 * @param pVmxTransient Pointer to the VMX transient structure.
677 */
678DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
679{
680 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
681 {
682 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
683 AssertRCReturn(rc, rc);
684 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
685 }
686 return VINF_SUCCESS;
687}
688
689
690/**
691 * Reads the VM-exit instruction length field from the VMCS into the VMX
692 * transient structure.
693 *
694 * @returns VBox status code.
695 * @param pVmxTransient Pointer to the VMX transient structure.
696 */
697DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
698{
699 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
700 {
701 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
702 AssertRCReturn(rc, rc);
703 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_LEN;
704 }
705 return VINF_SUCCESS;
706}
707
708
709/**
710 * Reads the VM-exit instruction-information field from the VMCS into
711 * the VMX transient structure.
712 *
713 * @returns VBox status code.
714 * @param pVmxTransient Pointer to the VMX transient structure.
715 */
716DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
717{
718 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
719 {
720 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
721 AssertRCReturn(rc, rc);
722 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_INFO;
723 }
724 return VINF_SUCCESS;
725}
726
727
728/**
729 * Reads the VM-exit Qualification from the VMCS into the VMX transient structure.
730 *
731 * @returns VBox status code.
732 * @param pVCpu The cross context virtual CPU structure of the
733 * calling EMT. (Required for the VMCS cache case.)
734 * @param pVmxTransient Pointer to the VMX transient structure.
735 */
736DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
737{
738 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
739 {
740 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
741 AssertRCReturn(rc, rc);
742 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION;
743 }
744 return VINF_SUCCESS;
745}
746
747
748/**
749 * Reads the Guest-linear address from the VMCS into the VMX transient structure.
750 *
751 * @returns VBox status code.
752 * @param pVCpu The cross context virtual CPU structure of the
753 * calling EMT. (Required for the VMCS cache case.)
754 * @param pVmxTransient Pointer to the VMX transient structure.
755 */
756DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
757{
758 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
759 {
760 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
761 AssertRCReturn(rc, rc);
762 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_GUEST_LINEAR_ADDR;
763 }
764 return VINF_SUCCESS;
765}
766
767
768/**
769 * Reads the IDT-vectoring information field from the VMCS into the VMX
770 * transient structure.
771 *
772 * @returns VBox status code.
773 * @param pVmxTransient Pointer to the VMX transient structure.
774 *
775 * @remarks No-long-jump zone!!!
776 */
777DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
778{
779 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
780 {
781 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
782 AssertRCReturn(rc, rc);
783 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_INFO;
784 }
785 return VINF_SUCCESS;
786}
787
788
789/**
790 * Reads the IDT-vectoring error code from the VMCS into the VMX
791 * transient structure.
792 *
793 * @returns VBox status code.
794 * @param pVmxTransient Pointer to the VMX transient structure.
795 */
796DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
797{
798 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
799 {
800 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
801 AssertRCReturn(rc, rc);
802 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
803 }
804 return VINF_SUCCESS;
805}
806
807
808/**
809 * Enters VMX root mode operation on the current CPU.
810 *
811 * @returns VBox status code.
812 * @param pVM The cross context VM structure. Can be
813 * NULL, after a resume.
814 * @param HCPhysCpuPage Physical address of the VMXON region.
815 * @param pvCpuPage Pointer to the VMXON region.
816 */
817static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
818{
819 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
820 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
821 Assert(pvCpuPage);
822 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
823
824 if (pVM)
825 {
826 /* Write the VMCS revision dword to the VMXON region. */
827 *(uint32_t *)pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
828 }
829
830 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
831 RTCCUINTREG fEFlags = ASMIntDisableFlags();
832
833 /* Enable the VMX bit in CR4 if necessary. */
834 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
835
836 /* Enter VMX root mode. */
837 int rc = VMXEnable(HCPhysCpuPage);
838 if (RT_FAILURE(rc))
839 {
840 if (!(uOldCr4 & X86_CR4_VMXE))
841 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
842
843 if (pVM)
844 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
845 }
846
847 /* Restore interrupts. */
848 ASMSetFlags(fEFlags);
849 return rc;
850}
851
852
853/**
854 * Exits VMX root mode operation on the current CPU.
855 *
856 * @returns VBox status code.
857 */
858static int hmR0VmxLeaveRootMode(void)
859{
860 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
861
862 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
863 RTCCUINTREG fEFlags = ASMIntDisableFlags();
864
865 /* If we're for some reason not in VMX root mode, then don't leave it. */
866 RTCCUINTREG uHostCR4 = ASMGetCR4();
867
868 int rc;
869 if (uHostCR4 & X86_CR4_VMXE)
870 {
871 /* Exit VMX root mode and clear the VMX bit in CR4. */
872 VMXDisable();
873 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
874 rc = VINF_SUCCESS;
875 }
876 else
877 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
878
879 /* Restore interrupts. */
880 ASMSetFlags(fEFlags);
881 return rc;
882}
883
884
885/**
886 * Allocates and maps one physically contiguous page. The allocated page is
887 * zero'd out. (Used by various VT-x structures).
888 *
889 * @returns IPRT status code.
890 * @param pMemObj Pointer to the ring-0 memory object.
891 * @param ppVirt Where to store the virtual address of the
892 * allocation.
893 * @param pHCPhys Where to store the physical address of the
894 * allocation.
895 */
896static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
897{
898 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
899 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
900 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
901
902 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
903 if (RT_FAILURE(rc))
904 return rc;
905 *ppVirt = RTR0MemObjAddress(*pMemObj);
906 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
907 ASMMemZero32(*ppVirt, PAGE_SIZE);
908 return VINF_SUCCESS;
909}
910
911
912/**
913 * Frees and unmaps an allocated physical page.
914 *
915 * @param pMemObj Pointer to the ring-0 memory object.
916 * @param ppVirt Where to re-initialize the virtual address of
917 * allocation as 0.
918 * @param pHCPhys Where to re-initialize the physical address of the
919 * allocation as 0.
920 */
921static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
922{
923 AssertPtr(pMemObj);
924 AssertPtr(ppVirt);
925 AssertPtr(pHCPhys);
926 if (*pMemObj != NIL_RTR0MEMOBJ)
927 {
928 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
929 AssertRC(rc);
930 *pMemObj = NIL_RTR0MEMOBJ;
931 *ppVirt = 0;
932 *pHCPhys = 0;
933 }
934}
935
936
937/**
938 * Worker function to free VT-x related structures.
939 *
940 * @returns IPRT status code.
941 * @param pVM The cross context VM structure.
942 */
943static void hmR0VmxStructsFree(PVM pVM)
944{
945 for (VMCPUID i = 0; i < pVM->cCpus; i++)
946 {
947 PVMCPU pVCpu = &pVM->aCpus[i];
948 AssertPtr(pVCpu);
949
950 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
951 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
952
953 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
954 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
955
956 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
957 }
958
959 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
960#ifdef VBOX_WITH_CRASHDUMP_MAGIC
961 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
962#endif
963}
964
965
966/**
967 * Worker function to allocate VT-x related VM structures.
968 *
969 * @returns IPRT status code.
970 * @param pVM The cross context VM structure.
971 */
972static int hmR0VmxStructsAlloc(PVM pVM)
973{
974 /*
975 * Initialize members up-front so we can cleanup properly on allocation failure.
976 */
977#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
978 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
979 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
980 pVM->hm.s.vmx.HCPhys##a_Name = 0;
981
982#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
983 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
984 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
985 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
986
987#ifdef VBOX_WITH_CRASHDUMP_MAGIC
988 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
989#endif
990 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
991
992 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
993 for (VMCPUID i = 0; i < pVM->cCpus; i++)
994 {
995 PVMCPU pVCpu = &pVM->aCpus[i];
996 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
997 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
998 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
999 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
1000 }
1001#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
1002#undef VMXLOCAL_INIT_VM_MEMOBJ
1003
1004 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
1005 AssertReturnStmt(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE) <= PAGE_SIZE,
1006 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
1007 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
1008
1009 /*
1010 * Allocate all the VT-x structures.
1011 */
1012 int rc = VINF_SUCCESS;
1013#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1014 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1015 if (RT_FAILURE(rc))
1016 goto cleanup;
1017 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1018 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1019#endif
1020
1021 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1022 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1023 {
1024 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1025 &pVM->hm.s.vmx.HCPhysApicAccess);
1026 if (RT_FAILURE(rc))
1027 goto cleanup;
1028 }
1029
1030 /*
1031 * Initialize per-VCPU VT-x structures.
1032 */
1033 for (VMCPUID i = 0; i < pVM->cCpus; i++)
1034 {
1035 PVMCPU pVCpu = &pVM->aCpus[i];
1036 AssertPtr(pVCpu);
1037
1038 /* Allocate the VM control structure (VMCS). */
1039 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1040 if (RT_FAILURE(rc))
1041 goto cleanup;
1042
1043 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1044 if ( PDMHasApic(pVM)
1045 && (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1046 {
1047 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1048 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1049 if (RT_FAILURE(rc))
1050 goto cleanup;
1051 }
1052
1053 /*
1054 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1055 * transparent accesses of specific MSRs.
1056 *
1057 * If the condition for enabling MSR bitmaps changes here, don't forget to
1058 * update HMAreMsrBitmapsAvailable().
1059 */
1060 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1061 {
1062 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1063 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1064 if (RT_FAILURE(rc))
1065 goto cleanup;
1066 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1067 }
1068
1069 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1070 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1071 if (RT_FAILURE(rc))
1072 goto cleanup;
1073
1074 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1075 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1076 if (RT_FAILURE(rc))
1077 goto cleanup;
1078 }
1079
1080 return VINF_SUCCESS;
1081
1082cleanup:
1083 hmR0VmxStructsFree(pVM);
1084 return rc;
1085}
1086
1087
1088/**
1089 * Does global VT-x initialization (called during module initialization).
1090 *
1091 * @returns VBox status code.
1092 */
1093VMMR0DECL(int) VMXR0GlobalInit(void)
1094{
1095#ifdef HMVMX_USE_FUNCTION_TABLE
1096 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1097# ifdef VBOX_STRICT
1098 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1099 Assert(g_apfnVMExitHandlers[i]);
1100# endif
1101#endif
1102 return VINF_SUCCESS;
1103}
1104
1105
1106/**
1107 * Does global VT-x termination (called during module termination).
1108 */
1109VMMR0DECL(void) VMXR0GlobalTerm()
1110{
1111 /* Nothing to do currently. */
1112}
1113
1114
1115/**
1116 * Sets up and activates VT-x on the current CPU.
1117 *
1118 * @returns VBox status code.
1119 * @param pHostCpu Pointer to the global CPU info struct.
1120 * @param pVM The cross context VM structure. Can be
1121 * NULL after a host resume operation.
1122 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1123 * fEnabledByHost is @c true).
1124 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1125 * @a fEnabledByHost is @c true).
1126 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1127 * enable VT-x on the host.
1128 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1129 */
1130VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1131 void *pvMsrs)
1132{
1133 Assert(pHostCpu);
1134 Assert(pvMsrs);
1135 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1136
1137 /* Enable VT-x if it's not already enabled by the host. */
1138 if (!fEnabledByHost)
1139 {
1140 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1141 if (RT_FAILURE(rc))
1142 return rc;
1143 }
1144
1145 /*
1146 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1147 * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1148 * invalidated when flushing by VPID.
1149 */
1150 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1151 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1152 {
1153 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
1154 pHostCpu->fFlushAsidBeforeUse = false;
1155 }
1156 else
1157 pHostCpu->fFlushAsidBeforeUse = true;
1158
1159 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1160 ++pHostCpu->cTlbFlushes;
1161
1162 return VINF_SUCCESS;
1163}
1164
1165
1166/**
1167 * Deactivates VT-x on the current CPU.
1168 *
1169 * @returns VBox status code.
1170 * @param pHostCpu Pointer to the global CPU info struct.
1171 * @param pvCpuPage Pointer to the VMXON region.
1172 * @param HCPhysCpuPage Physical address of the VMXON region.
1173 *
1174 * @remarks This function should never be called when SUPR0EnableVTx() or
1175 * similar was used to enable VT-x on the host.
1176 */
1177VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1178{
1179 RT_NOREF3(pHostCpu, pvCpuPage, HCPhysCpuPage);
1180
1181 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1182 return hmR0VmxLeaveRootMode();
1183}
1184
1185
1186/**
1187 * Sets the permission bits for the specified MSR in the MSR bitmap.
1188 *
1189 * @param pVCpu The cross context virtual CPU structure.
1190 * @param uMsr The MSR value.
1191 * @param enmRead Whether reading this MSR causes a VM-exit.
1192 * @param enmWrite Whether writing this MSR causes a VM-exit.
1193 */
1194static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1195{
1196 int32_t iBit;
1197 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1198
1199 /*
1200 * MSR Layout:
1201 * Byte index MSR range Interpreted as
1202 * 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
1203 * 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
1204 * 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
1205 * 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
1206 *
1207 * A bit corresponding to an MSR within the above range causes a VM-exit
1208 * if the bit is 1 on executions of RDMSR/WRMSR.
1209 *
1210 * If an MSR falls out of the MSR range, it always cause a VM-exit.
1211 *
1212 * See Intel spec. 24.6.9 "MSR-Bitmap Address".
1213 */
1214 if (uMsr <= 0x00001fff)
1215 iBit = uMsr;
1216 else if (uMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1217 {
1218 iBit = uMsr - UINT32_C(0xc0000000);
1219 pbMsrBitmap += 0x400;
1220 }
1221 else
1222 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1223
1224 Assert(iBit <= 0x1fff);
1225 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1226 ASMBitSet(pbMsrBitmap, iBit);
1227 else
1228 ASMBitClear(pbMsrBitmap, iBit);
1229
1230 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1231 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1232 else
1233 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1234}
1235
1236
1237/**
1238 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1239 * area.
1240 *
1241 * @returns VBox status code.
1242 * @param pVCpu The cross context virtual CPU structure.
1243 * @param cMsrs The number of MSRs.
1244 */
1245static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1246{
1247 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1248 uint64_t const uVmxMiscMsr = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc;
1249 uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(uVmxMiscMsr);
1250 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1251 {
1252 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1253 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1254 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1255 }
1256
1257 /* Update number of guest MSRs to load/store across the world-switch. */
1258 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1259 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1260
1261 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1262 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1263 AssertRCReturn(rc, rc);
1264
1265 /* Update the VCPU's copy of the MSR count. */
1266 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1267
1268 return VINF_SUCCESS;
1269}
1270
1271
1272/**
1273 * Adds a new (or updates the value of an existing) guest/host MSR
1274 * pair to be swapped during the world-switch as part of the
1275 * auto-load/store MSR area in the VMCS.
1276 *
1277 * @returns VBox status code.
1278 * @param pVCpu The cross context virtual CPU structure.
1279 * @param uMsr The MSR.
1280 * @param uGuestMsrValue Value of the guest MSR.
1281 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1282 * necessary.
1283 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1284 * its value was updated. Optional, can be NULL.
1285 */
1286static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1287 bool *pfAddedAndUpdated)
1288{
1289 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1290 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1291 uint32_t i;
1292 for (i = 0; i < cMsrs; i++)
1293 {
1294 if (pGuestMsr->u32Msr == uMsr)
1295 break;
1296 pGuestMsr++;
1297 }
1298
1299 bool fAdded = false;
1300 if (i == cMsrs)
1301 {
1302 ++cMsrs;
1303 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1304 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1305
1306 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1307 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1308 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1309
1310 fAdded = true;
1311 }
1312
1313 /* Update the MSR values in the auto-load/store MSR area. */
1314 pGuestMsr->u32Msr = uMsr;
1315 pGuestMsr->u64Value = uGuestMsrValue;
1316
1317 /* Create/update the MSR slot in the host MSR area. */
1318 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1319 pHostMsr += i;
1320 pHostMsr->u32Msr = uMsr;
1321
1322 /*
1323 * Update the host MSR only when requested by the caller AND when we're
1324 * adding it to the auto-load/store area. Otherwise, it would have been
1325 * updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1326 */
1327 bool fUpdatedMsrValue = false;
1328 if ( fAdded
1329 && fUpdateHostMsr)
1330 {
1331 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1332 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1333 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1334 fUpdatedMsrValue = true;
1335 }
1336
1337 if (pfAddedAndUpdated)
1338 *pfAddedAndUpdated = fUpdatedMsrValue;
1339 return VINF_SUCCESS;
1340}
1341
1342
1343/**
1344 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1345 * auto-load/store MSR area in the VMCS.
1346 *
1347 * @returns VBox status code.
1348 * @param pVCpu The cross context virtual CPU structure.
1349 * @param uMsr The MSR.
1350 */
1351static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1352{
1353 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1354 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1355 for (uint32_t i = 0; i < cMsrs; i++)
1356 {
1357 /* Find the MSR. */
1358 if (pGuestMsr->u32Msr == uMsr)
1359 {
1360 /* If it's the last MSR, simply reduce the count. */
1361 if (i == cMsrs - 1)
1362 {
1363 --cMsrs;
1364 break;
1365 }
1366
1367 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1368 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1369 pLastGuestMsr += cMsrs - 1;
1370 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1371 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1372
1373 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1374 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1375 pLastHostMsr += cMsrs - 1;
1376 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1377 pHostMsr->u64Value = pLastHostMsr->u64Value;
1378 --cMsrs;
1379 break;
1380 }
1381 pGuestMsr++;
1382 }
1383
1384 /* Update the VMCS if the count changed (meaning the MSR was found). */
1385 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1386 {
1387 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1388 AssertRCReturn(rc, rc);
1389
1390 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1391 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1392 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1393
1394 Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1395 return VINF_SUCCESS;
1396 }
1397
1398 return VERR_NOT_FOUND;
1399}
1400
1401
1402/**
1403 * Checks if the specified guest MSR is part of the auto-load/store area in
1404 * the VMCS.
1405 *
1406 * @returns true if found, false otherwise.
1407 * @param pVCpu The cross context virtual CPU structure.
1408 * @param uMsr The MSR to find.
1409 */
1410static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1411{
1412 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1413 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1414
1415 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1416 {
1417 if (pGuestMsr->u32Msr == uMsr)
1418 return true;
1419 }
1420 return false;
1421}
1422
1423
1424/**
1425 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1426 *
1427 * @param pVCpu The cross context virtual CPU structure.
1428 *
1429 * @remarks No-long-jump zone!!!
1430 */
1431static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1432{
1433 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1434 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1435 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1436 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1437
1438 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1439 {
1440 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1441
1442 /*
1443 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1444 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1445 */
1446 if (pHostMsr->u32Msr == MSR_K6_EFER)
1447 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1448 else
1449 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1450 }
1451
1452 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1453}
1454
1455
1456/**
1457 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1458 * perform lazy restoration of the host MSRs while leaving VT-x.
1459 *
1460 * @param pVCpu The cross context virtual CPU structure.
1461 *
1462 * @remarks No-long-jump zone!!!
1463 */
1464static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1465{
1466 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1467
1468 /*
1469 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1470 */
1471 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1472 {
1473 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1474#if HC_ARCH_BITS == 64
1475 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1476 {
1477 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1478 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1479 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1480 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1481 }
1482#endif
1483 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1484 }
1485}
1486
1487
1488/**
1489 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1490 * lazily while leaving VT-x.
1491 *
1492 * @returns true if it does, false otherwise.
1493 * @param pVCpu The cross context virtual CPU structure.
1494 * @param uMsr The MSR to check.
1495 */
1496static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1497{
1498 NOREF(pVCpu);
1499#if HC_ARCH_BITS == 64
1500 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1501 {
1502 switch (uMsr)
1503 {
1504 case MSR_K8_LSTAR:
1505 case MSR_K6_STAR:
1506 case MSR_K8_SF_MASK:
1507 case MSR_K8_KERNEL_GS_BASE:
1508 return true;
1509 }
1510 }
1511#else
1512 RT_NOREF(pVCpu, uMsr);
1513#endif
1514 return false;
1515}
1516
1517
1518/**
1519 * Loads a set of guests MSRs to allow read/passthru to the guest.
1520 *
1521 * The name of this function is slightly confusing. This function does NOT
1522 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1523 * common prefix for functions dealing with "lazy restoration" of the shared
1524 * MSRs.
1525 *
1526 * @param pVCpu The cross context virtual CPU structure.
1527 *
1528 * @remarks No-long-jump zone!!!
1529 */
1530static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1531{
1532 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1533 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1534
1535 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1536#if HC_ARCH_BITS == 64
1537 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1538 {
1539 /*
1540 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1541 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1542 * we can skip a few MSR writes.
1543 *
1544 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1545 * guest MSR values in the guest-CPU context might be different to what's currently
1546 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1547 * CPU, see @bugref{8728}.
1548 */
1549 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1550 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1551 && pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1552 && pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1553 && pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1554 && pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1555 {
1556#ifdef VBOX_STRICT
1557 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1558 Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1559 Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1560 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
1561#endif
1562 }
1563 else
1564 {
1565 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1566 ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
1567 ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
1568 ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
1569 }
1570 }
1571#endif
1572 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1573}
1574
1575
1576/**
1577 * Performs lazy restoration of the set of host MSRs if they were previously
1578 * loaded with guest MSR values.
1579 *
1580 * @param pVCpu The cross context virtual CPU structure.
1581 *
1582 * @remarks No-long-jump zone!!!
1583 * @remarks The guest MSRs should have been saved back into the guest-CPU
1584 * context by hmR0VmxImportGuestState()!!!
1585 */
1586static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1587{
1588 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1589 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1590
1591 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1592 {
1593 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1594#if HC_ARCH_BITS == 64
1595 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1596 {
1597 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1598 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1599 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1600 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1601 }
1602#endif
1603 }
1604 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1605}
1606
1607
1608/**
1609 * Verifies that our cached values of the VMCS fields are all consistent with
1610 * what's actually present in the VMCS.
1611 *
1612 * @returns VBox status code.
1613 * @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
1614 * @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
1615 * VMCS content. HMCPU error-field is
1616 * updated, see VMX_VCI_XXX.
1617 * @param pVCpu The cross context virtual CPU structure.
1618 */
1619static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1620{
1621 uint32_t u32Val;
1622 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1623 AssertRCReturn(rc, rc);
1624 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32EntryCtls == u32Val,
1625 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1626 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
1627 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1628
1629 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1630 AssertRCReturn(rc, rc);
1631 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ExitCtls == u32Val,
1632 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1633 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
1634 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1635
1636 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1637 AssertRCReturn(rc, rc);
1638 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32PinCtls == u32Val,
1639 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1640 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
1641 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1642
1643 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1644 AssertRCReturn(rc, rc);
1645 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls == u32Val,
1646 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1647 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
1648 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1649
1650 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
1651 {
1652 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1653 AssertRCReturn(rc, rc);
1654 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1655 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1656 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
1657 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1658 }
1659
1660 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
1661 AssertRCReturn(rc, rc);
1662 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32XcptBitmap == u32Val,
1663 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap, u32Val),
1664 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
1665 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1666
1667 uint64_t u64Val;
1668 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
1669 AssertRCReturn(rc, rc);
1670 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u64TscOffset == u64Val,
1671 ("Cache=%#RX64 VMCS=%#RX64\n", pVCpu->hm.s.vmx.u64TscOffset, u64Val),
1672 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
1673 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1674
1675 return VINF_SUCCESS;
1676}
1677
1678
1679#ifdef VBOX_STRICT
1680/**
1681 * Verifies that our cached host EFER value has not changed
1682 * since we cached it.
1683 *
1684 * @param pVCpu The cross context virtual CPU structure.
1685 */
1686static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1687{
1688 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1689
1690 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
1691 {
1692 uint64_t u64Val;
1693 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1694 AssertRC(rc);
1695
1696 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1697 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1698 }
1699}
1700
1701
1702/**
1703 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1704 * VMCS are correct.
1705 *
1706 * @param pVCpu The cross context virtual CPU structure.
1707 */
1708static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1709{
1710 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1711
1712 /* Verify MSR counts in the VMCS are what we think it should be. */
1713 uint32_t cMsrs;
1714 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1715 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1716
1717 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1718 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1719
1720 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1721 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1722
1723 PCVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1724 PCVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1725 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1726 {
1727 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1728 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1729 pGuestMsr->u32Msr, cMsrs));
1730
1731 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1732 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1733 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1734
1735 /* Verify that the permissions are as expected in the MSR bitmap. */
1736 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1737 {
1738 VMXMSREXITREAD enmRead;
1739 VMXMSREXITWRITE enmWrite;
1740 rc = HMVmxGetMsrPermission(pVCpu->hm.s.vmx.pvMsrBitmap, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1741 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("HMVmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1742 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1743 {
1744 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1745 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1746 }
1747 else
1748 {
1749 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1750 pGuestMsr->u32Msr, cMsrs));
1751 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1752 pGuestMsr->u32Msr, cMsrs));
1753 }
1754 }
1755 }
1756}
1757#endif /* VBOX_STRICT */
1758
1759
1760/**
1761 * Flushes the TLB using EPT.
1762 *
1763 * @returns VBox status code.
1764 * @param pVCpu The cross context virtual CPU structure of the calling
1765 * EMT. Can be NULL depending on @a enmTlbFlush.
1766 * @param enmTlbFlush Type of flush.
1767 *
1768 * @remarks Caller is responsible for making sure this function is called only
1769 * when NestedPaging is supported and providing @a enmTlbFlush that is
1770 * supported by the CPU.
1771 * @remarks Can be called with interrupts disabled.
1772 */
1773static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush)
1774{
1775 uint64_t au64Descriptor[2];
1776 if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
1777 au64Descriptor[0] = 0;
1778 else
1779 {
1780 Assert(pVCpu);
1781 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1782 }
1783 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1784
1785 int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
1786 AssertMsg(rc == VINF_SUCCESS,
1787 ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0, rc));
1788
1789 if ( RT_SUCCESS(rc)
1790 && pVCpu)
1791 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1792}
1793
1794
1795/**
1796 * Flushes the TLB using VPID.
1797 *
1798 * @returns VBox status code.
1799 * @param pVCpu The cross context virtual CPU structure of the calling
1800 * EMT. Can be NULL depending on @a enmTlbFlush.
1801 * @param enmTlbFlush Type of flush.
1802 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1803 * on @a enmTlbFlush).
1804 *
1805 * @remarks Can be called with interrupts disabled.
1806 */
1807static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
1808{
1809 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
1810
1811 uint64_t au64Descriptor[2];
1812 if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
1813 {
1814 au64Descriptor[0] = 0;
1815 au64Descriptor[1] = 0;
1816 }
1817 else
1818 {
1819 AssertPtr(pVCpu);
1820 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1821 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1822 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1823 au64Descriptor[1] = GCPtr;
1824 }
1825
1826 int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
1827 AssertMsg(rc == VINF_SUCCESS,
1828 ("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1829
1830 if ( RT_SUCCESS(rc)
1831 && pVCpu)
1832 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1833 NOREF(rc);
1834}
1835
1836
1837/**
1838 * Invalidates a guest page by guest virtual address. Only relevant for
1839 * EPT/VPID, otherwise there is nothing really to invalidate.
1840 *
1841 * @returns VBox status code.
1842 * @param pVCpu The cross context virtual CPU structure.
1843 * @param GCVirt Guest virtual address of the page to invalidate.
1844 */
1845VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
1846{
1847 AssertPtr(pVCpu);
1848 LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
1849
1850 bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1851 if (!fFlushPending)
1852 {
1853 /*
1854 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1855 * the EPT case. See @bugref{6043} and @bugref{6177}.
1856 *
1857 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1858 * as this function maybe called in a loop with individual addresses.
1859 */
1860 PVM pVM = pVCpu->CTX_SUFF(pVM);
1861 if (pVM->hm.s.vmx.fVpid)
1862 {
1863 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1864
1865#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1866 /*
1867 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1868 * where executing INVVPID outside 64-bit mode does not flush translations of
1869 * 64-bit linear addresses, see @bugref{6208#c72}.
1870 */
1871 if (RT_HI_U32(GCVirt))
1872 fVpidFlush = false;
1873#endif
1874
1875 if (fVpidFlush)
1876 {
1877 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
1878 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1879 }
1880 else
1881 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1882 }
1883 else if (pVM->hm.s.fNestedPaging)
1884 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1885 }
1886
1887 return VINF_SUCCESS;
1888}
1889
1890
1891/**
1892 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1893 * case where neither EPT nor VPID is supported by the CPU.
1894 *
1895 * @param pVCpu The cross context virtual CPU structure.
1896 * @param pCpu Pointer to the global HM struct.
1897 *
1898 * @remarks Called with interrupts disabled.
1899 */
1900static void hmR0VmxFlushTaggedTlbNone(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1901{
1902 AssertPtr(pVCpu);
1903 AssertPtr(pCpu);
1904
1905 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1906
1907 Assert(pCpu->idCpu != NIL_RTCPUID);
1908 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1909 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1910 pVCpu->hm.s.fForceTLBFlush = false;
1911 return;
1912}
1913
1914
1915/**
1916 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1917 *
1918 * @param pVCpu The cross context virtual CPU structure.
1919 * @param pCpu Pointer to the global HM CPU struct.
1920 *
1921 * @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
1922 * nomenclature. The reason is, to avoid confusion in compare statements
1923 * since the host-CPU copies are named "ASID".
1924 *
1925 * @remarks Called with interrupts disabled.
1926 */
1927static void hmR0VmxFlushTaggedTlbBoth(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1928{
1929#ifdef VBOX_WITH_STATISTICS
1930 bool fTlbFlushed = false;
1931# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1932# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1933 if (!fTlbFlushed) \
1934 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1935 } while (0)
1936#else
1937# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1938# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1939#endif
1940
1941 AssertPtr(pCpu);
1942 AssertPtr(pVCpu);
1943 Assert(pCpu->idCpu != NIL_RTCPUID);
1944
1945 PVM pVM = pVCpu->CTX_SUFF(pVM);
1946 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1947 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1948 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1949
1950 /*
1951 * Force a TLB flush for the first world-switch if the current CPU differs from the one we
1952 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1953 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1954 * cannot reuse the current ASID anymore.
1955 */
1956 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1957 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1958 {
1959 ++pCpu->uCurrentAsid;
1960 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1961 {
1962 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1963 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1964 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1965 }
1966
1967 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1968 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1969 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1970
1971 /*
1972 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1973 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1974 */
1975 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1976 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1977 HMVMX_SET_TAGGED_TLB_FLUSHED();
1978 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1979 }
1980 else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
1981 {
1982 /*
1983 * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
1984 * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
1985 * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
1986 * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
1987 * mappings, see @bugref{6568}.
1988 *
1989 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
1990 */
1991 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1992 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1993 HMVMX_SET_TAGGED_TLB_FLUSHED();
1994 }
1995
1996 pVCpu->hm.s.fForceTLBFlush = false;
1997 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1998
1999 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2000 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2001 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2002 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2003 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2004 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2005 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2006 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2007 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2008
2009 /* Update VMCS with the VPID. */
2010 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2011 AssertRC(rc);
2012
2013#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2014}
2015
2016
2017/**
2018 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2019 *
2020 * @returns VBox status code.
2021 * @param pVCpu The cross context virtual CPU structure.
2022 * @param pCpu Pointer to the global HM CPU struct.
2023 *
2024 * @remarks Called with interrupts disabled.
2025 */
2026static void hmR0VmxFlushTaggedTlbEpt(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2027{
2028 AssertPtr(pVCpu);
2029 AssertPtr(pCpu);
2030 Assert(pCpu->idCpu != NIL_RTCPUID);
2031 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2032 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2033
2034 /*
2035 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2036 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2037 */
2038 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2039 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2040 {
2041 pVCpu->hm.s.fForceTLBFlush = true;
2042 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2043 }
2044
2045 /* Check for explicit TLB flushes. */
2046 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2047 {
2048 pVCpu->hm.s.fForceTLBFlush = true;
2049 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2050 }
2051
2052 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2053 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2054
2055 if (pVCpu->hm.s.fForceTLBFlush)
2056 {
2057 hmR0VmxFlushEpt(pVCpu, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2058 pVCpu->hm.s.fForceTLBFlush = false;
2059 }
2060}
2061
2062
2063/**
2064 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2065 *
2066 * @returns VBox status code.
2067 * @param pVCpu The cross context virtual CPU structure.
2068 * @param pCpu Pointer to the global HM CPU struct.
2069 *
2070 * @remarks Called with interrupts disabled.
2071 */
2072static void hmR0VmxFlushTaggedTlbVpid(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2073{
2074 AssertPtr(pVCpu);
2075 AssertPtr(pCpu);
2076 Assert(pCpu->idCpu != NIL_RTCPUID);
2077 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2078 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2079
2080 /*
2081 * Force a TLB flush for the first world switch if the current CPU differs from the one we
2082 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2083 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2084 * cannot reuse the current ASID anymore.
2085 */
2086 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2087 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2088 {
2089 pVCpu->hm.s.fForceTLBFlush = true;
2090 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2091 }
2092
2093 /* Check for explicit TLB flushes. */
2094 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2095 {
2096 /*
2097 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2098 * hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2099 * fExplicitFlush = true here and change the pCpu->fFlushAsidBeforeUse check below to
2100 * include fExplicitFlush's too) - an obscure corner case.
2101 */
2102 pVCpu->hm.s.fForceTLBFlush = true;
2103 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2104 }
2105
2106 PVM pVM = pVCpu->CTX_SUFF(pVM);
2107 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2108 if (pVCpu->hm.s.fForceTLBFlush)
2109 {
2110 ++pCpu->uCurrentAsid;
2111 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2112 {
2113 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2114 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2115 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2116 }
2117
2118 pVCpu->hm.s.fForceTLBFlush = false;
2119 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2120 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2121 if (pCpu->fFlushAsidBeforeUse)
2122 {
2123 if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2124 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2125 else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2126 {
2127 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2128 pCpu->fFlushAsidBeforeUse = false;
2129 }
2130 else
2131 {
2132 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2133 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2134 }
2135 }
2136 }
2137
2138 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2139 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2140 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2141 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2142 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2143 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2144 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2145
2146 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2147 AssertRC(rc);
2148}
2149
2150
2151/**
2152 * Flushes the guest TLB entry based on CPU capabilities.
2153 *
2154 * @param pVCpu The cross context virtual CPU structure.
2155 * @param pCpu Pointer to the global HM CPU struct.
2156 */
2157DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2158{
2159#ifdef HMVMX_ALWAYS_FLUSH_TLB
2160 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2161#endif
2162 PVM pVM = pVCpu->CTX_SUFF(pVM);
2163 switch (pVM->hm.s.vmx.enmTlbFlushType)
2164 {
2165 case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVCpu, pCpu); break;
2166 case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pVCpu, pCpu); break;
2167 case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pVCpu, pCpu); break;
2168 case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pVCpu, pCpu); break;
2169 default:
2170 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2171 break;
2172 }
2173 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2174}
2175
2176
2177/**
2178 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2179 * TLB entries from the host TLB before VM-entry.
2180 *
2181 * @returns VBox status code.
2182 * @param pVM The cross context VM structure.
2183 */
2184static int hmR0VmxSetupTaggedTlb(PVM pVM)
2185{
2186 /*
2187 * Determine optimal flush type for Nested Paging.
2188 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2189 * guest execution (see hmR3InitFinalizeR0()).
2190 */
2191 if (pVM->hm.s.fNestedPaging)
2192 {
2193 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2194 {
2195 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2196 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2197 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2198 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2199 else
2200 {
2201 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2202 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2203 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2204 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2205 }
2206
2207 /* Make sure the write-back cacheable memory type for EPT is supported. */
2208 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2209 {
2210 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2211 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2212 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2213 }
2214
2215 /* EPT requires a page-walk length of 4. */
2216 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2217 {
2218 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2219 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2220 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2221 }
2222 }
2223 else
2224 {
2225 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2226 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2227 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2228 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2229 }
2230 }
2231
2232 /*
2233 * Determine optimal flush type for VPID.
2234 */
2235 if (pVM->hm.s.vmx.fVpid)
2236 {
2237 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2238 {
2239 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2240 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2241 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2242 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2243 else
2244 {
2245 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2246 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2247 LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2248 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2249 LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2250 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2251 pVM->hm.s.vmx.fVpid = false;
2252 }
2253 }
2254 else
2255 {
2256 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2257 Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2258 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2259 pVM->hm.s.vmx.fVpid = false;
2260 }
2261 }
2262
2263 /*
2264 * Setup the handler for flushing tagged-TLBs.
2265 */
2266 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2267 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2268 else if (pVM->hm.s.fNestedPaging)
2269 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2270 else if (pVM->hm.s.vmx.fVpid)
2271 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2272 else
2273 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2274 return VINF_SUCCESS;
2275}
2276
2277
2278/**
2279 * Sets up pin-based VM-execution controls in the VMCS.
2280 *
2281 * @returns VBox status code.
2282 * @param pVCpu The cross context virtual CPU structure.
2283 *
2284 * @remarks We don't really care about optimizing vmwrites here as it's done only
2285 * once per VM and hence we don't care about VMCS-field cache comparisons.
2286 */
2287static int hmR0VmxSetupPinCtls(PVMCPU pVCpu)
2288{
2289 PVM pVM = pVCpu->CTX_SUFF(pVM);
2290 uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.disallowed0; /* Bits set here must always be set. */
2291 uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2292
2293 fVal |= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2294 | VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2295
2296 if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2297 fVal |= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2298
2299 /* Enable the VMX preemption timer. */
2300 if (pVM->hm.s.vmx.fUsePreemptTimer)
2301 {
2302 Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2303 fVal |= VMX_PIN_CTLS_PREEMPT_TIMER;
2304 }
2305
2306#if 0
2307 /* Enable posted-interrupt processing. */
2308 if (pVM->hm.s.fPostedIntrs)
2309 {
2310 Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2311 Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2312 fVal |= VMX_PIN_CTL_POSTED_INT;
2313 }
2314#endif
2315
2316 if ((fVal & fZap) != fVal)
2317 {
2318 LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2319 pVM->hm.s.vmx.Msrs.PinCtls.n.disallowed0, fVal, fZap));
2320 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2321 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2322 }
2323
2324 /* Commit it to the VMCS and update our cache. */
2325 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2326 AssertRCReturn(rc, rc);
2327 pVCpu->hm.s.vmx.u32PinCtls = fVal;
2328
2329 return VINF_SUCCESS;
2330}
2331
2332
2333/**
2334 * Sets up secondary processor-based VM-execution controls in the VMCS.
2335 *
2336 * @returns VBox status code.
2337 * @param pVCpu The cross context virtual CPU structure.
2338 *
2339 * @remarks We don't really care about optimizing vmwrites here as it's done only
2340 * once per VM and hence we don't care about VMCS-field cache comparisons.
2341 */
2342static int hmR0VmxSetupProcCtls2(PVMCPU pVCpu)
2343{
2344 PVM pVM = pVCpu->CTX_SUFF(pVM);
2345 uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2346 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2347
2348 /* WBINVD causes a VM-exit. */
2349 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2350 fVal |= VMX_PROC_CTLS2_WBINVD_EXIT;
2351
2352 /* Enable EPT (aka nested-paging). */
2353 if (pVM->hm.s.fNestedPaging)
2354 fVal |= VMX_PROC_CTLS2_EPT;
2355
2356 /*
2357 * Enable the INVPCID instruction if supported by the hardware and we expose
2358 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2359 */
2360 if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
2361 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2362 fVal |= VMX_PROC_CTLS2_INVPCID;
2363
2364 /* Enable VPID. */
2365 if (pVM->hm.s.vmx.fVpid)
2366 fVal |= VMX_PROC_CTLS2_VPID;
2367
2368 /* Enable Unrestricted guest execution. */
2369 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2370 fVal |= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
2371
2372#if 0
2373 if (pVM->hm.s.fVirtApicRegs)
2374 {
2375 /* Enable APIC-register virtualization. */
2376 Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
2377 fVal |= VMX_PROC_CTLS2_APIC_REG_VIRT;
2378
2379 /* Enable virtual-interrupt delivery. */
2380 Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
2381 fVal |= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
2382 }
2383#endif
2384
2385 /* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
2386 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2387 * done dynamically. */
2388 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
2389 {
2390 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2391 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2392 fVal |= VMX_PROC_CTLS2_VIRT_APIC_ACCESS; /* Virtualize APIC accesses. */
2393 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2394 AssertRCReturn(rc, rc);
2395 }
2396
2397 /* Enable RDTSCP. */
2398 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
2399 fVal |= VMX_PROC_CTLS2_RDTSCP;
2400
2401 /* Enable Pause-Loop exiting. */
2402 if ( pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT
2403 && pVM->hm.s.vmx.cPleGapTicks
2404 && pVM->hm.s.vmx.cPleWindowTicks)
2405 {
2406 fVal |= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
2407
2408 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2409 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2410 AssertRCReturn(rc, rc);
2411 }
2412
2413 if ((fVal & fZap) != fVal)
2414 {
2415 LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2416 pVM->hm.s.vmx.Msrs.ProcCtls2.n.disallowed0, fVal, fZap));
2417 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2418 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2419 }
2420
2421 /* Commit it to the VMCS and update our cache. */
2422 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2423 AssertRCReturn(rc, rc);
2424 pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2425
2426 return VINF_SUCCESS;
2427}
2428
2429
2430/**
2431 * Sets up processor-based VM-execution controls in the VMCS.
2432 *
2433 * @returns VBox status code.
2434 * @param pVCpu The cross context virtual CPU structure.
2435 *
2436 * @remarks We don't really care about optimizing vmwrites here as it's done only
2437 * once per VM and hence we don't care about VMCS-field cache comparisons.
2438 */
2439static int hmR0VmxSetupProcCtls(PVMCPU pVCpu)
2440{
2441 PVM pVM = pVCpu->CTX_SUFF(pVM);
2442 uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2443 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2444
2445 fVal |= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
2446 | VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2447 | VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2448 | VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2449 | VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2450 | VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2451 | VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2452
2453 /* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2454 if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
2455 || (pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
2456 {
2457 LogRelFunc(("Unsupported VMX_PROC_CTLS_MOV_DR_EXIT combo!"));
2458 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2459 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2460 }
2461
2462 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2463 if (!pVM->hm.s.fNestedPaging)
2464 {
2465 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2466 fVal |= VMX_PROC_CTLS_INVLPG_EXIT
2467 | VMX_PROC_CTLS_CR3_LOAD_EXIT
2468 | VMX_PROC_CTLS_CR3_STORE_EXIT;
2469 }
2470
2471 /* Use TPR shadowing if supported by the CPU. */
2472 if ( PDMHasApic(pVM)
2473 && pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
2474 {
2475 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2476 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2477 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2478 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2479 AssertRCReturn(rc, rc);
2480
2481 fVal |= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2482 /* CR8 writes cause a VM-exit based on TPR threshold. */
2483 Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
2484 Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
2485 }
2486 else
2487 {
2488 /*
2489 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2490 * Set this control only for 64-bit guests.
2491 */
2492 if (pVM->hm.s.fAllow64BitGuests)
2493 {
2494 fVal |= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2495 | VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2496 }
2497 }
2498
2499 /* Use MSR-bitmaps if supported by the CPU. */
2500 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2501 {
2502 fVal |= VMX_PROC_CTLS_USE_MSR_BITMAPS;
2503
2504 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2505 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2506 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2507 AssertRCReturn(rc, rc);
2508
2509 /*
2510 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2511 * automatically using dedicated fields in the VMCS.
2512 */
2513 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2514 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2515 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2516 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2517 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2518#if HC_ARCH_BITS == 64
2519 /*
2520 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2521 */
2522 if (pVM->hm.s.fAllow64BitGuests)
2523 {
2524 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2525 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2526 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2527 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2528 }
2529#endif
2530 /*
2531 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2532 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2533 */
2534 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2535 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2536
2537 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2538 }
2539
2540 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2541 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2542 fVal |= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
2543
2544 if ((fVal & fZap) != fVal)
2545 {
2546 LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2547 pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0, fVal, fZap));
2548 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2549 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2550 }
2551
2552 /* Commit it to the VMCS and update our cache. */
2553 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2554 AssertRCReturn(rc, rc);
2555 pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2556
2557 /* Set up secondary processor-based VM-execution controls if the CPU supports it. */
2558 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2559 return hmR0VmxSetupProcCtls2(pVCpu);
2560
2561 /* Sanity check, should not really happen. */
2562 if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2563 {
2564 LogRelFunc(("Unrestricted Guest enabled when secondary processor-based VM-execution controls not available\n"));
2565 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2566 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2567 }
2568
2569 /* Old CPUs without secondary processor-based VM-execution controls would end up here. */
2570 return VINF_SUCCESS;
2571}
2572
2573
2574/**
2575 * Sets up miscellaneous (everything other than Pin & Processor-based
2576 * VM-execution) control fields in the VMCS.
2577 *
2578 * @returns VBox status code.
2579 * @param pVCpu The cross context virtual CPU structure.
2580 */
2581static int hmR0VmxSetupMiscCtls(PVMCPU pVCpu)
2582{
2583 AssertPtr(pVCpu);
2584
2585 int rc = VERR_GENERAL_FAILURE;
2586
2587 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2588#if 0
2589 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2590 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2591 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2592
2593 /*
2594 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2595 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2596 * We thus use the exception bitmap to control it rather than use both.
2597 */
2598 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2599 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2600
2601 /* All IO & IOIO instructions cause VM-exits. */
2602 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2603 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2604
2605 /* Initialize the MSR-bitmap area. */
2606 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2607 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2608 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2609 AssertRCReturn(rc, rc);
2610#endif
2611
2612 /* Setup MSR auto-load/store area. */
2613 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2614 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2615 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2616 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2617 AssertRCReturn(rc, rc);
2618
2619 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2620 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2621 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2622 AssertRCReturn(rc, rc);
2623
2624 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2625 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2626 AssertRCReturn(rc, rc);
2627
2628 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2629#if 0
2630 /* Setup debug controls */
2631 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2632 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2633 AssertRCReturn(rc, rc);
2634#endif
2635
2636 return rc;
2637}
2638
2639
2640/**
2641 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2642 *
2643 * We shall setup those exception intercepts that don't change during the
2644 * lifetime of the VM here. The rest are done dynamically while loading the
2645 * guest state.
2646 *
2647 * @returns VBox status code.
2648 * @param pVCpu The cross context virtual CPU structure.
2649 */
2650static int hmR0VmxInitXcptBitmap(PVMCPU pVCpu)
2651{
2652 AssertPtr(pVCpu);
2653
2654 uint32_t uXcptBitmap;
2655
2656 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2657 uXcptBitmap = RT_BIT_32(X86_XCPT_AC);
2658
2659 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2660 and writes, and because recursive #DBs can cause the CPU hang, we must always
2661 intercept #DB. */
2662 uXcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2663
2664 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2665 if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
2666 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
2667
2668 /* Commit it to the VMCS. */
2669 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2670 AssertRCReturn(rc, rc);
2671
2672 /* Update our cache of the exception bitmap. */
2673 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
2674 return VINF_SUCCESS;
2675}
2676
2677
2678/**
2679 * Does per-VM VT-x initialization.
2680 *
2681 * @returns VBox status code.
2682 * @param pVM The cross context VM structure.
2683 */
2684VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2685{
2686 LogFlowFunc(("pVM=%p\n", pVM));
2687
2688 int rc = hmR0VmxStructsAlloc(pVM);
2689 if (RT_FAILURE(rc))
2690 {
2691 LogRelFunc(("hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2692 return rc;
2693 }
2694
2695 return VINF_SUCCESS;
2696}
2697
2698
2699/**
2700 * Does per-VM VT-x termination.
2701 *
2702 * @returns VBox status code.
2703 * @param pVM The cross context VM structure.
2704 */
2705VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2706{
2707 LogFlowFunc(("pVM=%p\n", pVM));
2708
2709#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2710 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2711 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2712#endif
2713 hmR0VmxStructsFree(pVM);
2714 return VINF_SUCCESS;
2715}
2716
2717
2718/**
2719 * Sets up the VM for execution under VT-x.
2720 * This function is only called once per-VM during initialization.
2721 *
2722 * @returns VBox status code.
2723 * @param pVM The cross context VM structure.
2724 */
2725VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2726{
2727 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2728 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2729
2730 LogFlowFunc(("pVM=%p\n", pVM));
2731
2732 /*
2733 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be
2734 * allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2735 * pRealModeTSS, see hmR3InitFinalizeR0Intel().
2736 */
2737 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2738 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2739 || !pVM->hm.s.vmx.pRealModeTSS))
2740 {
2741 LogRelFunc(("Invalid real-on-v86 state.\n"));
2742 return VERR_INTERNAL_ERROR;
2743 }
2744
2745 /* Initialize these always, see hmR3InitFinalizeR0().*/
2746 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
2747 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
2748
2749 /* Setup the tagged-TLB flush handlers. */
2750 int rc = hmR0VmxSetupTaggedTlb(pVM);
2751 if (RT_FAILURE(rc))
2752 {
2753 LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2754 return rc;
2755 }
2756
2757 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2758 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2759#if HC_ARCH_BITS == 64
2760 if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
2761 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2762 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
2763 {
2764 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2765 }
2766#endif
2767
2768 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2769 RTCCUINTREG const uHostCR4 = ASMGetCR4();
2770 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2771 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2772
2773 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2774 {
2775 PVMCPU pVCpu = &pVM->aCpus[i];
2776 AssertPtr(pVCpu);
2777 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2778
2779 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2780 Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2781
2782 /* Set revision dword at the beginning of the VMCS structure. */
2783 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
2784
2785 /* Set the VMCS launch state to "clear", see Intel spec. 31.6 "Preparation and launch a virtual machine". */
2786 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2787 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc\n", rc),
2788 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2789
2790 /* Load this VMCS as the current VMCS. */
2791 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2792 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc\n", rc),
2793 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2794
2795 rc = hmR0VmxSetupPinCtls(pVCpu);
2796 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc\n", rc),
2797 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2798
2799 rc = hmR0VmxSetupProcCtls(pVCpu);
2800 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc\n", rc),
2801 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2802
2803 rc = hmR0VmxSetupMiscCtls(pVCpu);
2804 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc\n", rc),
2805 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2806
2807 rc = hmR0VmxInitXcptBitmap(pVCpu);
2808 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc\n", rc),
2809 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2810
2811#if HC_ARCH_BITS == 32
2812 rc = hmR0VmxInitVmcsReadCache(pVCpu);
2813 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc\n", rc),
2814 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2815#endif
2816
2817 /* Sync any CPU internal VMCS data back into our VMCS in memory. */
2818 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2819 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc\n", rc),
2820 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2821
2822 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2823
2824 hmR0VmxUpdateErrorRecord(pVCpu, rc);
2825 }
2826
2827 return VINF_SUCCESS;
2828}
2829
2830
2831/**
2832 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2833 * the VMCS.
2834 *
2835 * @returns VBox status code.
2836 */
2837static int hmR0VmxExportHostControlRegs(void)
2838{
2839 RTCCUINTREG uReg = ASMGetCR0();
2840 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2841 AssertRCReturn(rc, rc);
2842
2843 uReg = ASMGetCR3();
2844 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2845 AssertRCReturn(rc, rc);
2846
2847 uReg = ASMGetCR4();
2848 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2849 AssertRCReturn(rc, rc);
2850 return rc;
2851}
2852
2853
2854/**
2855 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2856 * the host-state area in the VMCS.
2857 *
2858 * @returns VBox status code.
2859 * @param pVCpu The cross context virtual CPU structure.
2860 */
2861static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2862{
2863#if HC_ARCH_BITS == 64
2864/**
2865 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2866 * requirements. See hmR0VmxExportHostSegmentRegs().
2867 */
2868# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2869 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2870 { \
2871 bool fValidSelector = true; \
2872 if ((selValue) & X86_SEL_LDT) \
2873 { \
2874 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2875 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2876 } \
2877 if (fValidSelector) \
2878 { \
2879 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2880 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2881 } \
2882 (selValue) = 0; \
2883 }
2884
2885 /*
2886 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2887 * should -not- save the messed up state without restoring the original host-state,
2888 * see @bugref{7240}.
2889 *
2890 * This apparently can happen (most likely the FPU changes), deal with it rather than
2891 * asserting. Was observed booting Solaris 10u10 32-bit guest.
2892 */
2893 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2894 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2895 {
2896 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2897 pVCpu->idCpu));
2898 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2899 }
2900 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2901#else
2902 RT_NOREF(pVCpu);
2903#endif
2904
2905 /*
2906 * Host DS, ES, FS and GS segment registers.
2907 */
2908#if HC_ARCH_BITS == 64
2909 RTSEL uSelDS = ASMGetDS();
2910 RTSEL uSelES = ASMGetES();
2911 RTSEL uSelFS = ASMGetFS();
2912 RTSEL uSelGS = ASMGetGS();
2913#else
2914 RTSEL uSelDS = 0;
2915 RTSEL uSelES = 0;
2916 RTSEL uSelFS = 0;
2917 RTSEL uSelGS = 0;
2918#endif
2919
2920 /*
2921 * Host CS and SS segment registers.
2922 */
2923 RTSEL uSelCS = ASMGetCS();
2924 RTSEL uSelSS = ASMGetSS();
2925
2926 /*
2927 * Host TR segment register.
2928 */
2929 RTSEL uSelTR = ASMGetTR();
2930
2931#if HC_ARCH_BITS == 64
2932 /*
2933 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2934 * gain VM-entry and restore them before we get preempted.
2935 *
2936 * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2937 */
2938 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2939 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2940 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2941 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2942# undef VMXLOCAL_ADJUST_HOST_SEG
2943#endif
2944
2945 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2946 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2947 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2948 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2949 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2950 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2951 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2952 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2953 Assert(uSelCS);
2954 Assert(uSelTR);
2955
2956 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2957#if 0
2958 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE))
2959 Assert(uSelSS != 0);
2960#endif
2961
2962 /* Write these host selector fields into the host-state area in the VMCS. */
2963 int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2964 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2965#if HC_ARCH_BITS == 64
2966 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2967 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2968 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2969 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2970#else
2971 NOREF(uSelDS);
2972 NOREF(uSelES);
2973 NOREF(uSelFS);
2974 NOREF(uSelGS);
2975#endif
2976 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2977 AssertRCReturn(rc, rc);
2978
2979 /*
2980 * Host GDTR and IDTR.
2981 */
2982 RTGDTR Gdtr;
2983 RTIDTR Idtr;
2984 RT_ZERO(Gdtr);
2985 RT_ZERO(Idtr);
2986 ASMGetGDTR(&Gdtr);
2987 ASMGetIDTR(&Idtr);
2988 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2989 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2990 AssertRCReturn(rc, rc);
2991
2992#if HC_ARCH_BITS == 64
2993 /*
2994 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
2995 * them to the maximum limit (0xffff) on every VM-exit.
2996 */
2997 if (Gdtr.cbGdt != 0xffff)
2998 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2999
3000 /*
3001 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
3002 * Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
3003 * as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
3004 * However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
3005 * uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
3006 * alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
3007 * at 0xffff on hosts where we are sure it won't cause trouble.
3008 */
3009# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3010 if (Idtr.cbIdt < 0x0fff)
3011# else
3012 if (Idtr.cbIdt != 0xffff)
3013# endif
3014 {
3015 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3016 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3017 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3018 }
3019#endif
3020
3021 /*
3022 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3023 * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3024 * RPL should be too in most cases.
3025 */
3026 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3027 ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3028
3029 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3030#if HC_ARCH_BITS == 64
3031 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3032
3033 /*
3034 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3035 * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3036 * restoration if the host has something else. Task switching is not supported in 64-bit
3037 * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3038 * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3039 *
3040 * [1] See Intel spec. 3.5 "System Descriptor Types".
3041 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3042 */
3043 PVM pVM = pVCpu->CTX_SUFF(pVM);
3044 Assert(pDesc->System.u4Type == 11);
3045 if ( pDesc->System.u16LimitLow != 0x67
3046 || pDesc->System.u4LimitHigh)
3047 {
3048 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3049 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3050 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3051 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3052 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3053 }
3054
3055 /*
3056 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3057 */
3058 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3059 {
3060 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3061 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3062 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3063 {
3064 /* The GDT is read-only but the writable GDT is available. */
3065 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3066 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3067 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3068 AssertRCReturn(rc, rc);
3069 }
3070 }
3071#else
3072 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3073#endif
3074 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3075 AssertRCReturn(rc, rc);
3076
3077 /*
3078 * Host FS base and GS base.
3079 */
3080#if HC_ARCH_BITS == 64
3081 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3082 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3083 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3084 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3085 AssertRCReturn(rc, rc);
3086
3087 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3088 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3089 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3090 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3091 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3092#endif
3093 return VINF_SUCCESS;
3094}
3095
3096
3097/**
3098 * Exports certain host MSRs in the VM-exit MSR-load area and some in the
3099 * host-state area of the VMCS.
3100 *
3101 * Theses MSRs will be automatically restored on the host after every successful
3102 * VM-exit.
3103 *
3104 * @returns VBox status code.
3105 * @param pVCpu The cross context virtual CPU structure.
3106 *
3107 * @remarks No-long-jump zone!!!
3108 */
3109static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3110{
3111 AssertPtr(pVCpu);
3112 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3113
3114 /*
3115 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3116 * rather than swapping them on every VM-entry.
3117 */
3118 hmR0VmxLazySaveHostMsrs(pVCpu);
3119
3120 /*
3121 * Host Sysenter MSRs.
3122 */
3123 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3124#if HC_ARCH_BITS == 32
3125 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3126 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3127#else
3128 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3129 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3130#endif
3131 AssertRCReturn(rc, rc);
3132
3133 /*
3134 * Host EFER MSR.
3135 *
3136 * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3137 * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3138 */
3139 PVM pVM = pVCpu->CTX_SUFF(pVM);
3140 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3141 {
3142 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3143 AssertRCReturn(rc, rc);
3144 }
3145
3146 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3147
3148 return VINF_SUCCESS;
3149}
3150
3151
3152/**
3153 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3154 *
3155 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3156 * these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3157 * hmR0VMxExportGuestEntryCtls().
3158 *
3159 * @returns true if we need to load guest EFER, false otherwise.
3160 * @param pVCpu The cross context virtual CPU structure.
3161 *
3162 * @remarks Requires EFER, CR4.
3163 * @remarks No-long-jump zone!!!
3164 */
3165static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu)
3166{
3167#ifdef HMVMX_ALWAYS_SWAP_EFER
3168 RT_NOREF(pVCpu);
3169 return true;
3170#else
3171
3172 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3173#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3174 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3175 if (CPUMIsGuestInLongModeEx(pCtx))
3176 return false;
3177#endif
3178
3179 PVM pVM = pVCpu->CTX_SUFF(pVM);
3180 uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3181 uint64_t const u64GuestEfer = pCtx->msrEFER;
3182
3183 /*
3184 * For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3185 * guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3186 */
3187 if ( CPUMIsGuestInLongModeEx(pCtx)
3188 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3189 {
3190 return true;
3191 }
3192
3193 /*
3194 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3195 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3196 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3197 */
3198 if ( (pCtx->cr4 & X86_CR4_PAE)
3199 && (pCtx->cr0 & X86_CR0_PG)
3200 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3201 {
3202 /* Assert that host is NX capable. */
3203 Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
3204 return true;
3205 }
3206
3207 return false;
3208#endif
3209}
3210
3211
3212/**
3213 * Exports the guest state with appropriate VM-entry controls in the VMCS.
3214 *
3215 * These controls can affect things done on VM-exit; e.g. "load debug controls",
3216 * see Intel spec. 24.8.1 "VM-entry controls".
3217 *
3218 * @returns VBox status code.
3219 * @param pVCpu The cross context virtual CPU structure.
3220 *
3221 * @remarks Requires EFER.
3222 * @remarks No-long-jump zone!!!
3223 */
3224static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu)
3225{
3226 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3227 {
3228 PVM pVM = pVCpu->CTX_SUFF(pVM);
3229 uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
3230 uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3231
3232 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3233 fVal |= VMX_ENTRY_CTLS_LOAD_DEBUG;
3234
3235 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3236 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
3237 {
3238 fVal |= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
3239 Log4Func(("VMX_ENTRY_CTLS_IA32E_MODE_GUEST\n"));
3240 }
3241 else
3242 Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
3243
3244 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3245 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3246 && hmR0VmxShouldSwapEferMsr(pVCpu))
3247 {
3248 fVal |= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
3249 Log4Func(("VMX_ENTRY_CTLS_LOAD_EFER_MSR\n"));
3250 }
3251
3252 /*
3253 * The following should -not- be set (since we're not in SMM mode):
3254 * - VMX_ENTRY_CTLS_ENTRY_TO_SMM
3255 * - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
3256 */
3257
3258 /** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
3259 * VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
3260
3261 if ((fVal & fZap) != fVal)
3262 {
3263 Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3264 pVM->hm.s.vmx.Msrs.EntryCtls.n.disallowed0, fVal, fZap));
3265 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3266 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3267 }
3268
3269 /* Commit it to the VMCS and update our cache. */
3270 if (pVCpu->hm.s.vmx.u32EntryCtls != fVal)
3271 {
3272 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3273 AssertRCReturn(rc, rc);
3274 pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3275 }
3276
3277 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3278 }
3279 return VINF_SUCCESS;
3280}
3281
3282
3283/**
3284 * Exports the guest state with appropriate VM-exit controls in the VMCS.
3285 *
3286 * @returns VBox status code.
3287 * @param pVCpu The cross context virtual CPU structure.
3288 *
3289 * @remarks Requires EFER.
3290 */
3291static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu)
3292{
3293 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3294 {
3295 PVM pVM = pVCpu->CTX_SUFF(pVM);
3296 uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
3297 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3298
3299 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3300 fVal |= VMX_EXIT_CTLS_SAVE_DEBUG;
3301
3302 /*
3303 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3304 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3305 * hmR0VmxExportHostMsrs().
3306 */
3307#if HC_ARCH_BITS == 64
3308 fVal |= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
3309 Log4Func(("VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE\n"));
3310#else
3311 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3312 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3313 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3314 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3315 {
3316 /* The switcher returns to long mode, EFER is managed by the switcher. */
3317 fVal |= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
3318 Log4Func(("VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE\n"));
3319 }
3320 else
3321 Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
3322#endif
3323
3324 /* If the newer VMCS fields for managing EFER exists, use it. */
3325 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3326 && hmR0VmxShouldSwapEferMsr(pVCpu))
3327 {
3328 fVal |= VMX_EXIT_CTLS_SAVE_EFER_MSR
3329 | VMX_EXIT_CTLS_LOAD_EFER_MSR;
3330 Log4Func(("VMX_EXIT_CTLS_SAVE_EFER_MSR and VMX_EXIT_CTLS_LOAD_EFER_MSR\n"));
3331 }
3332
3333 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3334 Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
3335
3336 /** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
3337 * VMX_EXIT_CTLS_SAVE_PAT_MSR,
3338 * VMX_EXIT_CTLS_LOAD_PAT_MSR. */
3339
3340 /* Enable saving of the VMX preemption timer value on VM-exit. */
3341 if ( pVM->hm.s.vmx.fUsePreemptTimer
3342 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
3343 fVal |= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
3344
3345 if ((fVal & fZap) != fVal)
3346 {
3347 LogRelFunc(("Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3348 pVM->hm.s.vmx.Msrs.ExitCtls.n.disallowed0, fVal, fZap));
3349 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3350 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3351 }
3352
3353 /* Commit it to the VMCS and update our cache. */
3354 if (pVCpu->hm.s.vmx.u32ExitCtls != fVal)
3355 {
3356 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3357 AssertRCReturn(rc, rc);
3358 pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3359 }
3360
3361 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3362 }
3363 return VINF_SUCCESS;
3364}
3365
3366
3367/**
3368 * Sets the TPR threshold in the VMCS.
3369 *
3370 * @returns VBox status code.
3371 * @param pVCpu The cross context virtual CPU structure.
3372 * @param u32TprThreshold The TPR threshold (task-priority class only).
3373 */
3374DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3375{
3376 Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
3377 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3378 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3379}
3380
3381
3382/**
3383 * Exports the guest APIC TPR state into the VMCS.
3384 *
3385 * @returns VBox status code.
3386 * @param pVCpu The cross context virtual CPU structure.
3387 *
3388 * @remarks No-long-jump zone!!!
3389 */
3390static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3391{
3392 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3393 {
3394 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
3395
3396 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3397 && APICIsEnabled(pVCpu))
3398 {
3399 /*
3400 * Setup TPR shadowing.
3401 */
3402 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
3403 {
3404 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3405
3406 bool fPendingIntr = false;
3407 uint8_t u8Tpr = 0;
3408 uint8_t u8PendingIntr = 0;
3409 int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3410 AssertRCReturn(rc, rc);
3411
3412 /*
3413 * If there are interrupts pending but masked by the TPR, instruct VT-x to
3414 * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3415 * priority of the pending interrupt so we can deliver the interrupt. If there
3416 * are no interrupts pending, set threshold to 0 to not cause any
3417 * TPR-below-threshold VM-exits.
3418 */
3419 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3420 uint32_t u32TprThreshold = 0;
3421 if (fPendingIntr)
3422 {
3423 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3424 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3425 const uint8_t u8TprPriority = u8Tpr >> 4;
3426 if (u8PendingPriority <= u8TprPriority)
3427 u32TprThreshold = u8PendingPriority;
3428 }
3429
3430 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3431 AssertRCReturn(rc, rc);
3432 }
3433 }
3434 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3435 }
3436 return VINF_SUCCESS;
3437}
3438
3439
3440/**
3441 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3442 *
3443 * @returns Guest's interruptibility-state.
3444 * @param pVCpu The cross context virtual CPU structure.
3445 *
3446 * @remarks No-long-jump zone!!!
3447 */
3448static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu)
3449{
3450 /*
3451 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3452 */
3453 uint32_t fIntrState = 0;
3454 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3455 {
3456 /* If inhibition is active, RIP & RFLAGS should've been accessed
3457 (i.e. read previously from the VMCS or from ring-3). */
3458 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3459#ifdef VBOX_STRICT
3460 uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
3461 AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3462#endif
3463 if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3464 {
3465 if (pCtx->eflags.Bits.u1IF)
3466 fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
3467 else
3468 fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
3469 }
3470 else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3471 {
3472 /*
3473 * We can clear the inhibit force flag as even if we go back to the recompiler
3474 * without executing guest code in VT-x, the flag's condition to be cleared is
3475 * met and thus the cleared state is correct.
3476 */
3477 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3478 }
3479 }
3480
3481 /*
3482 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3483 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3484 * setting this would block host-NMIs and IRET will not clear the blocking.
3485 *
3486 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3487 */
3488 if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS)
3489 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
3490 {
3491 fIntrState |= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
3492 }
3493
3494 return fIntrState;
3495}
3496
3497
3498/**
3499 * Exports the guest's interruptibility-state into the guest-state area in the
3500 * VMCS.
3501 *
3502 * @returns VBox status code.
3503 * @param pVCpu The cross context virtual CPU structure.
3504 * @param fIntrState The interruptibility-state to set.
3505 */
3506static int hmR0VmxExportGuestIntrState(PVMCPU pVCpu, uint32_t fIntrState)
3507{
3508 NOREF(pVCpu);
3509 AssertMsg(!(fIntrState & 0xfffffff0), ("%#x\n", fIntrState)); /* Bits 31:4 MBZ. */
3510 Assert((fIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3511 return VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
3512}
3513
3514
3515/**
3516 * Exports the exception intercepts required for guest execution in the VMCS.
3517 *
3518 * @returns VBox status code.
3519 * @param pVCpu The cross context virtual CPU structure.
3520 *
3521 * @remarks No-long-jump zone!!!
3522 */
3523static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3524{
3525 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3526 {
3527 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3528
3529 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3530 if (pVCpu->hm.s.fGIMTrapXcptUD)
3531 uXcptBitmap |= RT_BIT(X86_XCPT_UD);
3532#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3533 else
3534 uXcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3535#endif
3536
3537 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_AC));
3538 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_DB));
3539
3540 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3541 {
3542 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3543 AssertRCReturn(rc, rc);
3544 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3545 }
3546
3547 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3548 Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", uXcptBitmap));
3549 }
3550 return VINF_SUCCESS;
3551}
3552
3553
3554/**
3555 * Exports the guest's RIP into the guest-state area in the VMCS.
3556 *
3557 * @returns VBox status code.
3558 * @param pVCpu The cross context virtual CPU structure.
3559 *
3560 * @remarks No-long-jump zone!!!
3561 */
3562static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
3563{
3564 int rc = VINF_SUCCESS;
3565 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3566 {
3567 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
3568
3569 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
3570 AssertRCReturn(rc, rc);
3571
3572 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3573 Log4Func(("RIP=%#RX64\n", pVCpu->cpum.GstCtx.rip));
3574 }
3575 return rc;
3576}
3577
3578
3579/**
3580 * Exports the guest's RSP into the guest-state area in the VMCS.
3581 *
3582 * @returns VBox status code.
3583 * @param pVCpu The cross context virtual CPU structure.
3584 *
3585 * @remarks No-long-jump zone!!!
3586 */
3587static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
3588{
3589 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3590 {
3591 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
3592
3593 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
3594 AssertRCReturn(rc, rc);
3595
3596 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3597 }
3598 return VINF_SUCCESS;
3599}
3600
3601
3602/**
3603 * Exports the guest's RFLAGS into the guest-state area in the VMCS.
3604 *
3605 * @returns VBox status code.
3606 * @param pVCpu The cross context virtual CPU structure.
3607 *
3608 * @remarks No-long-jump zone!!!
3609 */
3610static int hmR0VmxExportGuestRflags(PVMCPU pVCpu)
3611{
3612 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3613 {
3614 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
3615
3616 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3617 Let us assert it as such and use 32-bit VMWRITE. */
3618 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
3619 X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
3620 Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3621 Assert(!(fEFlags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3622
3623 /*
3624 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3625 * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3626 * can run the real-mode guest code under Virtual 8086 mode.
3627 */
3628 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3629 {
3630 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3631 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3632 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3633 fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3634 fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3635 }
3636
3637 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3638 AssertRCReturn(rc, rc);
3639
3640 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3641 Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3642 }
3643 return VINF_SUCCESS;
3644}
3645
3646
3647/**
3648 * Exports the guest CR0 control register into the guest-state area in the VMCS.
3649 *
3650 * The guest FPU state is always pre-loaded hence we don't need to bother about
3651 * sharing FPU related CR0 bits between the guest and host.
3652 *
3653 * @returns VBox status code.
3654 * @param pVCpu The cross context virtual CPU structure.
3655 *
3656 * @remarks No-long-jump zone!!!
3657 */
3658static int hmR0VmxExportGuestCR0(PVMCPU pVCpu)
3659{
3660 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3661 {
3662 PVM pVM = pVCpu->CTX_SUFF(pVM);
3663 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
3664 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.cr0));
3665
3666 uint32_t const u32ShadowCr0 = pVCpu->cpum.GstCtx.cr0;
3667 uint32_t u32GuestCr0 = pVCpu->cpum.GstCtx.cr0;
3668
3669 /*
3670 * Setup VT-x's view of the guest CR0.
3671 * Minimize VM-exits due to CR3 changes when we have NestedPaging.
3672 */
3673 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3674 if (pVM->hm.s.fNestedPaging)
3675 {
3676 if (CPUMIsGuestPagingEnabled(pVCpu))
3677 {
3678 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3679 uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
3680 | VMX_PROC_CTLS_CR3_STORE_EXIT);
3681 }
3682 else
3683 {
3684 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3685 uProcCtls |= VMX_PROC_CTLS_CR3_LOAD_EXIT
3686 | VMX_PROC_CTLS_CR3_STORE_EXIT;
3687 }
3688
3689 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3690 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3691 uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
3692 }
3693 else
3694 {
3695 /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3696 u32GuestCr0 |= X86_CR0_WP;
3697 }
3698
3699 /*
3700 * Guest FPU bits.
3701 *
3702 * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3703 * using CR0.TS.
3704 *
3705 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3706 * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3707 */
3708 u32GuestCr0 |= X86_CR0_NE;
3709
3710 /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3711 bool const fInterceptMF = !(u32ShadowCr0 & X86_CR0_NE);
3712
3713 /*
3714 * Update exception intercepts.
3715 */
3716 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3717 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3718 {
3719 Assert(PDMVmmDevHeapIsEnabled(pVM));
3720 Assert(pVM->hm.s.vmx.pRealModeTSS);
3721 uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3722 }
3723 else
3724 {
3725 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3726 uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3727 if (fInterceptMF)
3728 uXcptBitmap |= RT_BIT(X86_XCPT_MF);
3729 }
3730
3731 /* Additional intercepts for debugging, define these yourself explicitly. */
3732#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3733 uXcptBitmap |= 0
3734 | RT_BIT(X86_XCPT_BP)
3735 | RT_BIT(X86_XCPT_DE)
3736 | RT_BIT(X86_XCPT_NM)
3737 | RT_BIT(X86_XCPT_TS)
3738 | RT_BIT(X86_XCPT_UD)
3739 | RT_BIT(X86_XCPT_NP)
3740 | RT_BIT(X86_XCPT_SS)
3741 | RT_BIT(X86_XCPT_GP)
3742 | RT_BIT(X86_XCPT_PF)
3743 | RT_BIT(X86_XCPT_MF)
3744 ;
3745#elif defined(HMVMX_ALWAYS_TRAP_PF)
3746 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
3747#endif
3748 Assert(pVM->hm.s.fNestedPaging || (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
3749
3750 /*
3751 * Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3752 */
3753 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3754 uint32_t fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3755 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3756 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
3757 else
3758 Assert((fSetCr0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3759
3760 u32GuestCr0 |= fSetCr0;
3761 u32GuestCr0 &= fZapCr0;
3762 u32GuestCr0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3763
3764 /*
3765 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3766 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3767 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3768 */
3769 uint32_t u32Cr0Mask = X86_CR0_PE
3770 | X86_CR0_NE
3771 | (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3772 | X86_CR0_PG
3773 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3774 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3775 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3776
3777 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3778 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3779 * and @bugref{6944}. */
3780#if 0
3781 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3782 u32Cr0Mask &= ~X86_CR0_PE;
3783#endif
3784 /*
3785 * Finally, update VMCS fields with the CR0 values and the exception bitmap.
3786 */
3787 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCr0);
3788 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32ShadowCr0);
3789 if (u32Cr0Mask != pVCpu->hm.s.vmx.u32Cr0Mask)
3790 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32Cr0Mask);
3791 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3792 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3793 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3794 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3795 AssertRCReturn(rc, rc);
3796
3797 /* Update our caches. */
3798 pVCpu->hm.s.vmx.u32Cr0Mask = u32Cr0Mask;
3799 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3800 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3801
3802 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3803
3804 Log4Func(("u32Cr0Mask=%#RX32 u32ShadowCr0=%#RX32 u32GuestCr0=%#RX32 (fSetCr0=%#RX32 fZapCr0=%#RX32\n", u32Cr0Mask,
3805 u32ShadowCr0, u32GuestCr0, fSetCr0, fZapCr0));
3806 }
3807
3808 return VINF_SUCCESS;
3809}
3810
3811
3812/**
3813 * Exports the guest control registers (CR3, CR4) into the guest-state area
3814 * in the VMCS.
3815 *
3816 * @returns VBox strict status code.
3817 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3818 * without unrestricted guest access and the VMMDev is not presently
3819 * mapped (e.g. EFI32).
3820 *
3821 * @param pVCpu The cross context virtual CPU structure.
3822 *
3823 * @remarks No-long-jump zone!!!
3824 */
3825static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu)
3826{
3827 int rc = VINF_SUCCESS;
3828 PVM pVM = pVCpu->CTX_SUFF(pVM);
3829
3830 /*
3831 * Guest CR2.
3832 * It's always loaded in the assembler code. Nothing to do here.
3833 */
3834
3835 /*
3836 * Guest CR3.
3837 */
3838 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3839 {
3840 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
3841
3842 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3843 if (pVM->hm.s.fNestedPaging)
3844 {
3845 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3846
3847 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3848 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3849 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3850 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3851
3852 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3853 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3854 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3855
3856 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3857 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3858 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3859 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3860 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3861 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3862 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3863
3864 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3865 AssertRCReturn(rc, rc);
3866
3867 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3868 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3869 || CPUMIsGuestPagingEnabledEx(pCtx))
3870 {
3871 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3872 if (CPUMIsGuestInPAEModeEx(pCtx))
3873 {
3874 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3875 AssertRCReturn(rc, rc);
3876 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3877 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3878 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3879 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3880 AssertRCReturn(rc, rc);
3881 }
3882
3883 /*
3884 * The guest's view of its CR3 is unblemished with Nested Paging when the
3885 * guest is using paging or we have unrestricted guest execution to handle
3886 * the guest when it's not using paging.
3887 */
3888 GCPhysGuestCR3 = pCtx->cr3;
3889 }
3890 else
3891 {
3892 /*
3893 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
3894 * thinks it accesses physical memory directly, we use our identity-mapped
3895 * page table to map guest-linear to guest-physical addresses. EPT takes care
3896 * of translating it to host-physical addresses.
3897 */
3898 RTGCPHYS GCPhys;
3899 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3900
3901 /* We obtain it here every time as the guest could have relocated this PCI region. */
3902 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3903 if (RT_SUCCESS(rc))
3904 { /* likely */ }
3905 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3906 {
3907 Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3908 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3909 }
3910 else
3911 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3912
3913 GCPhysGuestCR3 = GCPhys;
3914 }
3915
3916 Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
3917 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3918 AssertRCReturn(rc, rc);
3919 }
3920 else
3921 {
3922 /* Non-nested paging case, just use the hypervisor's CR3. */
3923 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3924
3925 Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
3926 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3927 AssertRCReturn(rc, rc);
3928 }
3929
3930 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3931 }
3932
3933 /*
3934 * Guest CR4.
3935 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3936 */
3937 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3938 {
3939 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3940 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
3941 Assert(!RT_HI_U32(pCtx->cr4));
3942
3943 uint32_t u32GuestCr4 = pCtx->cr4;
3944 uint32_t const u32ShadowCr4 = pCtx->cr4;
3945
3946 /*
3947 * Setup VT-x's view of the guest CR4.
3948 *
3949 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3950 * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3951 * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3952 *
3953 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3954 */
3955 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3956 {
3957 Assert(pVM->hm.s.vmx.pRealModeTSS);
3958 Assert(PDMVmmDevHeapIsEnabled(pVM));
3959 u32GuestCr4 &= ~X86_CR4_VME;
3960 }
3961
3962 if (pVM->hm.s.fNestedPaging)
3963 {
3964 if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3965 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3966 {
3967 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3968 u32GuestCr4 |= X86_CR4_PSE;
3969 /* Our identity mapping is a 32-bit page directory. */
3970 u32GuestCr4 &= ~X86_CR4_PAE;
3971 }
3972 /* else use guest CR4.*/
3973 }
3974 else
3975 {
3976 /*
3977 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3978 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3979 */
3980 switch (pVCpu->hm.s.enmShadowMode)
3981 {
3982 case PGMMODE_REAL: /* Real-mode. */
3983 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3984 case PGMMODE_32_BIT: /* 32-bit paging. */
3985 {
3986 u32GuestCr4 &= ~X86_CR4_PAE;
3987 break;
3988 }
3989
3990 case PGMMODE_PAE: /* PAE paging. */
3991 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3992 {
3993 u32GuestCr4 |= X86_CR4_PAE;
3994 break;
3995 }
3996
3997 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3998 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3999#ifdef VBOX_ENABLE_64_BITS_GUESTS
4000 break;
4001#endif
4002 default:
4003 AssertFailed();
4004 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4005 }
4006 }
4007
4008 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4009 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4010 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4011 u32GuestCr4 |= fSetCr4;
4012 u32GuestCr4 &= fZapCr4;
4013
4014 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them,
4015 that would cause a VM-exit. */
4016 uint32_t u32Cr4Mask = X86_CR4_VME
4017 | X86_CR4_PAE
4018 | X86_CR4_PGE
4019 | X86_CR4_PSE
4020 | X86_CR4_VMXE;
4021 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4022 u32Cr4Mask |= X86_CR4_OSXSAVE;
4023 if (pVM->cpum.ro.GuestFeatures.fPcid)
4024 u32Cr4Mask |= X86_CR4_PCIDE;
4025
4026 /* Write VT-x's view of the guest CR4, the CR4 modify mask and the read-only CR4 shadow
4027 into the VMCS and update our cache. */
4028 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCr4);
4029 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32ShadowCr4);
4030 if (pVCpu->hm.s.vmx.u32Cr4Mask != u32Cr4Mask)
4031 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32Cr4Mask);
4032 AssertRCReturn(rc, rc);
4033 pVCpu->hm.s.vmx.u32Cr4Mask = u32Cr4Mask;
4034
4035 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4036 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
4037
4038 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4039
4040 Log4Func(("u32GuestCr4=%#RX32 u32ShadowCr4=%#RX32 (fSetCr4=%#RX32 fZapCr4=%#RX32)\n", u32GuestCr4, u32ShadowCr4, fSetCr4,
4041 fZapCr4));
4042 }
4043 return rc;
4044}
4045
4046
4047/**
4048 * Exports the guest debug registers into the guest-state area in the VMCS.
4049 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4050 *
4051 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4052 *
4053 * @returns VBox status code.
4054 * @param pVCpu The cross context virtual CPU structure.
4055 *
4056 * @remarks No-long-jump zone!!!
4057 */
4058static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu)
4059{
4060 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4061
4062#ifdef VBOX_STRICT
4063 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4064 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
4065 {
4066 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4067 Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0);
4068 Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
4069 }
4070#endif
4071
4072 bool fSteppingDB = false;
4073 bool fInterceptMovDRx = false;
4074 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4075 if (pVCpu->hm.s.fSingleInstruction)
4076 {
4077 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4078 PVM pVM = pVCpu->CTX_SUFF(pVM);
4079 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
4080 {
4081 uProcCtls |= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
4082 Assert(fSteppingDB == false);
4083 }
4084 else
4085 {
4086 pVCpu->cpum.GstCtx.eflags.u32 |= X86_EFL_TF;
4087 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
4088 pVCpu->hm.s.fClearTrapFlag = true;
4089 fSteppingDB = true;
4090 }
4091 }
4092
4093 uint32_t u32GuestDr7;
4094 if ( fSteppingDB
4095 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4096 {
4097 /*
4098 * Use the combined guest and host DRx values found in the hypervisor register set
4099 * because the debugger has breakpoints active or someone is single stepping on the
4100 * host side without a monitor trap flag.
4101 *
4102 * Note! DBGF expects a clean DR6 state before executing guest code.
4103 */
4104#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4105 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4106 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4107 {
4108 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4109 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4110 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4111 }
4112 else
4113#endif
4114 if (!CPUMIsHyperDebugStateActive(pVCpu))
4115 {
4116 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4117 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4118 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4119 }
4120
4121 /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4122 u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4123 pVCpu->hm.s.fUsingHyperDR7 = true;
4124 fInterceptMovDRx = true;
4125 }
4126 else
4127 {
4128 /*
4129 * If the guest has enabled debug registers, we need to load them prior to
4130 * executing guest code so they'll trigger at the right time.
4131 */
4132 if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD))
4133 {
4134#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4135 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4136 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4137 {
4138 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4139 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4140 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4141 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4142 }
4143 else
4144#endif
4145 if (!CPUMIsGuestDebugStateActive(pVCpu))
4146 {
4147 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4148 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4149 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4150 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4151 }
4152 Assert(!fInterceptMovDRx);
4153 }
4154 /*
4155 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4156 * must intercept #DB in order to maintain a correct DR6 guest value, and
4157 * because we need to intercept it to prevent nested #DBs from hanging the
4158 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4159 */
4160#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4161 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4162 && !CPUMIsGuestDebugStateActive(pVCpu))
4163#else
4164 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4165#endif
4166 {
4167 fInterceptMovDRx = true;
4168 }
4169
4170 /* Update DR7 with the actual guest value. */
4171 u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
4172 pVCpu->hm.s.fUsingHyperDR7 = false;
4173 }
4174
4175 if (fInterceptMovDRx)
4176 uProcCtls |= VMX_PROC_CTLS_MOV_DR_EXIT;
4177 else
4178 uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
4179
4180 /*
4181 * Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
4182 * monitor-trap flag and update our cache.
4183 */
4184 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4185 {
4186 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4187 AssertRCReturn(rc2, rc2);
4188 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4189 }
4190
4191 /*
4192 * Update guest DR7.
4193 */
4194 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
4195 AssertRCReturn(rc, rc);
4196
4197 return VINF_SUCCESS;
4198}
4199
4200
4201#ifdef VBOX_STRICT
4202/**
4203 * Strict function to validate segment registers.
4204 *
4205 * @param pVCpu The cross context virtual CPU structure.
4206 *
4207 * @remarks Will import guest CR0 on strict builds during validation of
4208 * segments.
4209 */
4210static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu)
4211{
4212 /*
4213 * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4214 *
4215 * The reason we check for attribute value 0 in this function and not just the unusable bit is
4216 * because hmR0VmxExportGuestSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4217 * and doesn't change the guest-context value.
4218 */
4219 PVM pVM = pVCpu->CTX_SUFF(pVM);
4220 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4221 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4222 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4223 && ( !CPUMIsGuestInRealModeEx(pCtx)
4224 && !CPUMIsGuestInV86ModeEx(pCtx)))
4225 {
4226 /* Protected mode checks */
4227 /* CS */
4228 Assert(pCtx->cs.Attr.n.u1Present);
4229 Assert(!(pCtx->cs.Attr.u & 0xf00));
4230 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4231 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4232 || !(pCtx->cs.Attr.n.u1Granularity));
4233 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4234 || (pCtx->cs.Attr.n.u1Granularity));
4235 /* CS cannot be loaded with NULL in protected mode. */
4236 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4237 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4238 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4239 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4240 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4241 else
4242 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4243 /* SS */
4244 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4245 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4246 if ( !(pCtx->cr0 & X86_CR0_PE)
4247 || pCtx->cs.Attr.n.u4Type == 3)
4248 {
4249 Assert(!pCtx->ss.Attr.n.u2Dpl);
4250 }
4251 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4252 {
4253 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4254 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4255 Assert(pCtx->ss.Attr.n.u1Present);
4256 Assert(!(pCtx->ss.Attr.u & 0xf00));
4257 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4258 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4259 || !(pCtx->ss.Attr.n.u1Granularity));
4260 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4261 || (pCtx->ss.Attr.n.u1Granularity));
4262 }
4263 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmentReg(). */
4264 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4265 {
4266 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4267 Assert(pCtx->ds.Attr.n.u1Present);
4268 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4269 Assert(!(pCtx->ds.Attr.u & 0xf00));
4270 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4271 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4272 || !(pCtx->ds.Attr.n.u1Granularity));
4273 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4274 || (pCtx->ds.Attr.n.u1Granularity));
4275 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4276 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4277 }
4278 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4279 {
4280 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4281 Assert(pCtx->es.Attr.n.u1Present);
4282 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4283 Assert(!(pCtx->es.Attr.u & 0xf00));
4284 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4285 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4286 || !(pCtx->es.Attr.n.u1Granularity));
4287 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4288 || (pCtx->es.Attr.n.u1Granularity));
4289 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4290 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4291 }
4292 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4293 {
4294 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4295 Assert(pCtx->fs.Attr.n.u1Present);
4296 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4297 Assert(!(pCtx->fs.Attr.u & 0xf00));
4298 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4299 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4300 || !(pCtx->fs.Attr.n.u1Granularity));
4301 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4302 || (pCtx->fs.Attr.n.u1Granularity));
4303 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4304 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4305 }
4306 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4307 {
4308 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4309 Assert(pCtx->gs.Attr.n.u1Present);
4310 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4311 Assert(!(pCtx->gs.Attr.u & 0xf00));
4312 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4313 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4314 || !(pCtx->gs.Attr.n.u1Granularity));
4315 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4316 || (pCtx->gs.Attr.n.u1Granularity));
4317 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4318 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4319 }
4320 /* 64-bit capable CPUs. */
4321# if HC_ARCH_BITS == 64
4322 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4323 Assert(!pCtx->ss.Attr.u || !RT_HI_U32(pCtx->ss.u64Base));
4324 Assert(!pCtx->ds.Attr.u || !RT_HI_U32(pCtx->ds.u64Base));
4325 Assert(!pCtx->es.Attr.u || !RT_HI_U32(pCtx->es.u64Base));
4326# endif
4327 }
4328 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4329 || ( CPUMIsGuestInRealModeEx(pCtx)
4330 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4331 {
4332 /* Real and v86 mode checks. */
4333 /* hmR0VmxExportGuestSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4334 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4335 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4336 {
4337 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4338 }
4339 else
4340 {
4341 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4342 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4343 }
4344
4345 /* CS */
4346 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4347 Assert(pCtx->cs.u32Limit == 0xffff);
4348 Assert(u32CSAttr == 0xf3);
4349 /* SS */
4350 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4351 Assert(pCtx->ss.u32Limit == 0xffff);
4352 Assert(u32SSAttr == 0xf3);
4353 /* DS */
4354 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4355 Assert(pCtx->ds.u32Limit == 0xffff);
4356 Assert(u32DSAttr == 0xf3);
4357 /* ES */
4358 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4359 Assert(pCtx->es.u32Limit == 0xffff);
4360 Assert(u32ESAttr == 0xf3);
4361 /* FS */
4362 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4363 Assert(pCtx->fs.u32Limit == 0xffff);
4364 Assert(u32FSAttr == 0xf3);
4365 /* GS */
4366 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4367 Assert(pCtx->gs.u32Limit == 0xffff);
4368 Assert(u32GSAttr == 0xf3);
4369 /* 64-bit capable CPUs. */
4370# if HC_ARCH_BITS == 64
4371 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4372 Assert(!u32SSAttr || !RT_HI_U32(pCtx->ss.u64Base));
4373 Assert(!u32DSAttr || !RT_HI_U32(pCtx->ds.u64Base));
4374 Assert(!u32ESAttr || !RT_HI_U32(pCtx->es.u64Base));
4375# endif
4376 }
4377}
4378#endif /* VBOX_STRICT */
4379
4380
4381/**
4382 * Exports a guest segment register into the guest-state area in the VMCS.
4383 *
4384 * @returns VBox status code.
4385 * @param pVCpu The cross context virtual CPU structure.
4386 * @param idxSel Index of the selector in the VMCS.
4387 * @param idxLimit Index of the segment limit in the VMCS.
4388 * @param idxBase Index of the segment base in the VMCS.
4389 * @param idxAccess Index of the access rights of the segment in the VMCS.
4390 * @param pSelReg Pointer to the segment selector.
4391 *
4392 * @remarks No-long-jump zone!!!
4393 */
4394static int hmR0VmxExportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
4395 PCCPUMSELREG pSelReg)
4396{
4397 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4398 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4399 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4400 AssertRCReturn(rc, rc);
4401
4402 uint32_t u32Access = pSelReg->Attr.u;
4403 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4404 {
4405 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4406 u32Access = 0xf3;
4407 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4408 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4409 }
4410 else
4411 {
4412 /*
4413 * The way to differentiate between whether this is really a null selector or was just
4414 * a selector loaded with 0 in real-mode is using the segment attributes. A selector
4415 * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4416 * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4417 * NULL selectors loaded in protected-mode have their attribute as 0.
4418 */
4419 if (!u32Access)
4420 u32Access = X86DESCATTR_UNUSABLE;
4421 }
4422
4423 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4424 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4425 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4426
4427 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4428 AssertRCReturn(rc, rc);
4429 return rc;
4430}
4431
4432
4433/**
4434 * Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4435 * into the guest-state area in the VMCS.
4436 *
4437 * @returns VBox status code.
4438 * @param pVCpu The cross context virtual CPU structure.
4439 *
4440 * @remarks Will import guest CR0 on strict builds during validation of
4441 * segments.
4442 * @remarks No-long-jump zone!!!
4443 */
4444static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu)
4445{
4446 int rc = VERR_INTERNAL_ERROR_5;
4447 PVM pVM = pVCpu->CTX_SUFF(pVM);
4448 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4449
4450 /*
4451 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4452 */
4453 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4454 {
4455#ifdef VBOX_WITH_REM
4456 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4457 {
4458 Assert(pVM->hm.s.vmx.pRealModeTSS);
4459 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4460 if ( pVCpu->hm.s.vmx.fWasInRealMode
4461 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4462 {
4463 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4464 in real-mode (e.g. OpenBSD 4.0) */
4465 REMFlushTBs(pVM);
4466 Log4Func(("Switch to protected mode detected!\n"));
4467 pVCpu->hm.s.vmx.fWasInRealMode = false;
4468 }
4469 }
4470#endif
4471 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
4472 {
4473 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
4474 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4475 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pCtx->cs.Attr.u;
4476 rc = HMVMX_EXPORT_SREG(CS, &pCtx->cs);
4477 AssertRCReturn(rc, rc);
4478 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
4479 }
4480
4481 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
4482 {
4483 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
4484 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4485 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pCtx->ss.Attr.u;
4486 rc = HMVMX_EXPORT_SREG(SS, &pCtx->ss);
4487 AssertRCReturn(rc, rc);
4488 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
4489 }
4490
4491 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
4492 {
4493 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
4494 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4495 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pCtx->ds.Attr.u;
4496 rc = HMVMX_EXPORT_SREG(DS, &pCtx->ds);
4497 AssertRCReturn(rc, rc);
4498 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
4499 }
4500
4501 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
4502 {
4503 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
4504 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4505 pVCpu->hm.s.vmx.RealMode.AttrES.u = pCtx->es.Attr.u;
4506 rc = HMVMX_EXPORT_SREG(ES, &pCtx->es);
4507 AssertRCReturn(rc, rc);
4508 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
4509 }
4510
4511 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
4512 {
4513 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
4514 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4515 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pCtx->fs.Attr.u;
4516 rc = HMVMX_EXPORT_SREG(FS, &pCtx->fs);
4517 AssertRCReturn(rc, rc);
4518 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
4519 }
4520
4521 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
4522 {
4523 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
4524 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4525 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pCtx->gs.Attr.u;
4526 rc = HMVMX_EXPORT_SREG(GS, &pCtx->gs);
4527 AssertRCReturn(rc, rc);
4528 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
4529 }
4530
4531#ifdef VBOX_STRICT
4532 hmR0VmxValidateSegmentRegs(pVCpu);
4533#endif
4534
4535 Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pCtx->cs.Sel, pCtx->cs.u64Base,
4536 pCtx->cs.u32Limit, pCtx->cs.Attr.u));
4537 }
4538
4539 /*
4540 * Guest TR.
4541 */
4542 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4543 {
4544 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
4545
4546 /*
4547 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
4548 * achieved using the interrupt redirection bitmap (all bits cleared to let the guest
4549 * handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4550 */
4551 uint16_t u16Sel = 0;
4552 uint32_t u32Limit = 0;
4553 uint64_t u64Base = 0;
4554 uint32_t u32AccessRights = 0;
4555
4556 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4557 {
4558 u16Sel = pCtx->tr.Sel;
4559 u32Limit = pCtx->tr.u32Limit;
4560 u64Base = pCtx->tr.u64Base;
4561 u32AccessRights = pCtx->tr.Attr.u;
4562 }
4563 else
4564 {
4565 Assert(pVM->hm.s.vmx.pRealModeTSS);
4566 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
4567
4568 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4569 RTGCPHYS GCPhys;
4570 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4571 AssertRCReturn(rc, rc);
4572
4573 X86DESCATTR DescAttr;
4574 DescAttr.u = 0;
4575 DescAttr.n.u1Present = 1;
4576 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4577
4578 u16Sel = 0;
4579 u32Limit = HM_VTX_TSS_SIZE;
4580 u64Base = GCPhys; /* in real-mode phys = virt. */
4581 u32AccessRights = DescAttr.u;
4582 }
4583
4584 /* Validate. */
4585 Assert(!(u16Sel & RT_BIT(2)));
4586 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4587 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4588 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4589 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4590 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4591 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4592 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4593 Assert( (u32Limit & 0xfff) == 0xfff
4594 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4595 Assert( !(pCtx->tr.u32Limit & 0xfff00000)
4596 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4597
4598 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4599 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4600 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4601 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4602 AssertRCReturn(rc, rc);
4603
4604 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4605 Log4Func(("TR base=%#RX64\n", pCtx->tr.u64Base));
4606 }
4607
4608 /*
4609 * Guest GDTR.
4610 */
4611 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4612 {
4613 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
4614
4615 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
4616 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
4617 AssertRCReturn(rc, rc);
4618
4619 /* Validate. */
4620 Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4621
4622 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4623 Log4Func(("GDTR base=%#RX64\n", pCtx->gdtr.pGdt));
4624 }
4625
4626 /*
4627 * Guest LDTR.
4628 */
4629 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4630 {
4631 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
4632
4633 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4634 uint32_t u32Access = 0;
4635 if (!pCtx->ldtr.Attr.u)
4636 u32Access = X86DESCATTR_UNUSABLE;
4637 else
4638 u32Access = pCtx->ldtr.Attr.u;
4639
4640 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
4641 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
4642 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4643 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
4644 AssertRCReturn(rc, rc);
4645
4646 /* Validate. */
4647 if (!(u32Access & X86DESCATTR_UNUSABLE))
4648 {
4649 Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4650 Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4651 Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4652 Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4653 Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4654 Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4655 Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
4656 || !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4657 Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
4658 || pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4659 }
4660
4661 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4662 Log4Func(("LDTR base=%#RX64\n", pCtx->ldtr.u64Base));
4663 }
4664
4665 /*
4666 * Guest IDTR.
4667 */
4668 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4669 {
4670 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
4671
4672 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
4673 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
4674 AssertRCReturn(rc, rc);
4675
4676 /* Validate. */
4677 Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4678
4679 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4680 Log4Func(("IDTR base=%#RX64\n", pCtx->idtr.pIdt));
4681 }
4682
4683 return VINF_SUCCESS;
4684}
4685
4686
4687/**
4688 * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4689 * areas.
4690 *
4691 * These MSRs will automatically be loaded to the host CPU on every successful
4692 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4693 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4694 * -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4695 *
4696 * Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4697 *
4698 * @returns VBox status code.
4699 * @param pVCpu The cross context virtual CPU structure.
4700 *
4701 * @remarks No-long-jump zone!!!
4702 */
4703static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu)
4704{
4705 AssertPtr(pVCpu);
4706 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4707
4708 /*
4709 * MSRs that we use the auto-load/store MSR area in the VMCS.
4710 * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4711 */
4712 PVM pVM = pVCpu->CTX_SUFF(pVM);
4713 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4714 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4715 {
4716 if (pVM->hm.s.fAllow64BitGuests)
4717 {
4718#if HC_ARCH_BITS == 32
4719 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS | CPUMCTX_EXTRN_KERNEL_GS_BASE);
4720
4721 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pCtx->msrLSTAR, false, NULL);
4722 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pCtx->msrSTAR, false, NULL);
4723 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pCtx->msrSFMASK, false, NULL);
4724 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, false, NULL);
4725 AssertRCReturn(rc, rc);
4726# ifdef LOG_ENABLED
4727 PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4728 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4729 Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4730# endif
4731#endif
4732 }
4733 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4734 }
4735
4736 /*
4737 * Guest Sysenter MSRs.
4738 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4739 * VM-exits on WRMSRs for these MSRs.
4740 */
4741 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4742 {
4743 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
4744
4745 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4746 {
4747 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
4748 AssertRCReturn(rc, rc);
4749 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4750 }
4751
4752 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4753 {
4754 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
4755 AssertRCReturn(rc, rc);
4756 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4757 }
4758
4759 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4760 {
4761 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
4762 AssertRCReturn(rc, rc);
4763 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4764 }
4765 }
4766
4767 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4768 {
4769 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
4770
4771 if (hmR0VmxShouldSwapEferMsr(pVCpu))
4772 {
4773 /*
4774 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4775 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4776 */
4777 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4778 {
4779 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
4780 AssertRCReturn(rc,rc);
4781 Log4Func(("EFER=%#RX64\n", pCtx->msrEFER));
4782 }
4783 else
4784 {
4785 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pCtx->msrEFER, false /* fUpdateHostMsr */,
4786 NULL /* pfAddedAndUpdated */);
4787 AssertRCReturn(rc, rc);
4788
4789 /* We need to intercept reads too, see @bugref{7386#c16}. */
4790 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
4791 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4792 Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pCtx->msrEFER,
4793 pVCpu->hm.s.vmx.cMsrs));
4794 }
4795 }
4796 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4797 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4798 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4799 }
4800
4801 return VINF_SUCCESS;
4802}
4803
4804
4805#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4806/**
4807 * Check if guest state allows safe use of 32-bit switcher again.
4808 *
4809 * Segment bases and protected mode structures must be 32-bit addressable
4810 * because the 32-bit switcher will ignore high dword when writing these VMCS
4811 * fields. See @bugref{8432} for details.
4812 *
4813 * @returns true if safe, false if must continue to use the 64-bit switcher.
4814 * @param pCtx Pointer to the guest-CPU context.
4815 *
4816 * @remarks No-long-jump zone!!!
4817 */
4818static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
4819{
4820 if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4821 if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4822 if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4823 if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4824 if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4825 if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4826 if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4827 if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4828 if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4829 if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4830
4831 /* All good, bases are 32-bit. */
4832 return true;
4833}
4834#endif
4835
4836
4837/**
4838 * Selects up the appropriate function to run guest code.
4839 *
4840 * @returns VBox status code.
4841 * @param pVCpu The cross context virtual CPU structure.
4842 *
4843 * @remarks No-long-jump zone!!!
4844 */
4845static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu)
4846{
4847 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4848 if (CPUMIsGuestInLongModeEx(pCtx))
4849 {
4850#ifndef VBOX_ENABLE_64_BITS_GUESTS
4851 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4852#endif
4853 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4854#if HC_ARCH_BITS == 32
4855 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4856 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4857 {
4858#ifdef VBOX_STRICT
4859 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4860 {
4861 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4862 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4863 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4864 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4865 | HM_CHANGED_VMX_ENTRY_CTLS
4866 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4867 }
4868#endif
4869 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4870
4871 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4872 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4873 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4874 Log4Func(("Selected 64-bit switcher\n"));
4875 }
4876#else
4877 /* 64-bit host. */
4878 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4879#endif
4880 }
4881 else
4882 {
4883 /* Guest is not in long mode, use the 32-bit handler. */
4884#if HC_ARCH_BITS == 32
4885 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4886 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4887 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4888 {
4889# ifdef VBOX_STRICT
4890 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4891 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4892 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4893 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4894 | HM_CHANGED_VMX_ENTRY_CTLS
4895 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4896# endif
4897 }
4898# ifdef VBOX_ENABLE_64_BITS_GUESTS
4899 /*
4900 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4901 * design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4902 * switcher flag because now we know the guest is in a sane state where it's safe
4903 * to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4904 * the much faster 32-bit switcher again.
4905 */
4906 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4907 {
4908 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4909 Log4Func(("Selected 32-bit switcher\n"));
4910 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4911 }
4912 else
4913 {
4914 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4915 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4916 || hmR0VmxIs32BitSwitcherSafe(pCtx))
4917 {
4918 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4919 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4920 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4921 | HM_CHANGED_VMX_ENTRY_CTLS
4922 | HM_CHANGED_VMX_EXIT_CTLS
4923 | HM_CHANGED_HOST_CONTEXT);
4924 Log4Func(("Selected 32-bit switcher (safe)\n"));
4925 }
4926 }
4927# else
4928 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4929# endif
4930#else
4931 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4932#endif
4933 }
4934 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4935 return VINF_SUCCESS;
4936}
4937
4938
4939/**
4940 * Wrapper for running the guest code in VT-x.
4941 *
4942 * @returns VBox status code, no informational status codes.
4943 * @param pVCpu The cross context virtual CPU structure.
4944 *
4945 * @remarks No-long-jump zone!!!
4946 */
4947DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu)
4948{
4949 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4950 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4951 pCtx->fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
4952
4953 /*
4954 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4955 * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4956 * callee-saved and thus the need for this XMM wrapper.
4957 *
4958 * See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4959 */
4960 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4961 /** @todo Add stats for resume vs launch. */
4962 PVM pVM = pVCpu->CTX_SUFF(pVM);
4963#ifdef VBOX_WITH_KERNEL_USING_XMM
4964 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4965#else
4966 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4967#endif
4968 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4969 return rc;
4970}
4971
4972
4973/**
4974 * Reports world-switch error and dumps some useful debug info.
4975 *
4976 * @param pVCpu The cross context virtual CPU structure.
4977 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4978 * @param pVmxTransient Pointer to the VMX transient structure (only
4979 * exitReason updated).
4980 */
4981static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
4982{
4983 Assert(pVCpu);
4984 Assert(pVmxTransient);
4985 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
4986
4987 Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
4988 switch (rcVMRun)
4989 {
4990 case VERR_VMX_INVALID_VMXON_PTR:
4991 AssertFailed();
4992 break;
4993 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4994 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4995 {
4996 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4997 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4998 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
4999 AssertRC(rc);
5000
5001 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5002 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5003 Cannot do it here as we may have been long preempted. */
5004
5005#ifdef VBOX_STRICT
5006 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5007 pVmxTransient->uExitReason));
5008 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
5009 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5010 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5011 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5012 else
5013 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5014 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5015 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5016
5017 /* VMX control bits. */
5018 uint32_t u32Val;
5019 uint64_t u64Val;
5020 RTHCUINTREG uHCReg;
5021 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5022 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5023 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5024 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5025 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
5026 {
5027 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5028 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5029 }
5030 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5031 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5032 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5033 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5034 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5035 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5036 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5037 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5038 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5039 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5040 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5041 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5042 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5043 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5044 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5045 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5046 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5047 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5048 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5049 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5050 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5051 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5052 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5053 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5054 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5055 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5056 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5057 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5058 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5059 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5060 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5061 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5062 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5063 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5064 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5065 {
5066 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5067 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5068 }
5069
5070 /* Guest bits. */
5071 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5072 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
5073 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5074 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
5075 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5076 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
5077 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
5078 {
5079 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5080 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5081 }
5082
5083 /* Host bits. */
5084 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5085 Log4(("Host CR0 %#RHr\n", uHCReg));
5086 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5087 Log4(("Host CR3 %#RHr\n", uHCReg));
5088 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5089 Log4(("Host CR4 %#RHr\n", uHCReg));
5090
5091 RTGDTR HostGdtr;
5092 PCX86DESCHC pDesc;
5093 ASMGetGDTR(&HostGdtr);
5094 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5095 Log4(("Host CS %#08x\n", u32Val));
5096 if (u32Val < HostGdtr.cbGdt)
5097 {
5098 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5099 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5100 }
5101
5102 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5103 Log4(("Host DS %#08x\n", u32Val));
5104 if (u32Val < HostGdtr.cbGdt)
5105 {
5106 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5107 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5108 }
5109
5110 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5111 Log4(("Host ES %#08x\n", u32Val));
5112 if (u32Val < HostGdtr.cbGdt)
5113 {
5114 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5115 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5116 }
5117
5118 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5119 Log4(("Host FS %#08x\n", u32Val));
5120 if (u32Val < HostGdtr.cbGdt)
5121 {
5122 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5123 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5124 }
5125
5126 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5127 Log4(("Host GS %#08x\n", u32Val));
5128 if (u32Val < HostGdtr.cbGdt)
5129 {
5130 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5131 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5132 }
5133
5134 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5135 Log4(("Host SS %#08x\n", u32Val));
5136 if (u32Val < HostGdtr.cbGdt)
5137 {
5138 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5139 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5140 }
5141
5142 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5143 Log4(("Host TR %#08x\n", u32Val));
5144 if (u32Val < HostGdtr.cbGdt)
5145 {
5146 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5147 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5148 }
5149
5150 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5151 Log4(("Host TR Base %#RHv\n", uHCReg));
5152 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5153 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5154 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5155 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5156 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5157 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5158 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5159 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5160 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5161 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5162 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5163 Log4(("Host RSP %#RHv\n", uHCReg));
5164 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5165 Log4(("Host RIP %#RHv\n", uHCReg));
5166# if HC_ARCH_BITS == 64
5167 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5168 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5169 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5170 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5171 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5172 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5173# endif
5174#endif /* VBOX_STRICT */
5175 break;
5176 }
5177
5178 default:
5179 /* Impossible */
5180 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5181 break;
5182 }
5183}
5184
5185
5186#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5187#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5188# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5189#endif
5190#ifdef VBOX_STRICT
5191static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5192{
5193 switch (idxField)
5194 {
5195 case VMX_VMCS_GUEST_RIP:
5196 case VMX_VMCS_GUEST_RSP:
5197 case VMX_VMCS_GUEST_SYSENTER_EIP:
5198 case VMX_VMCS_GUEST_SYSENTER_ESP:
5199 case VMX_VMCS_GUEST_GDTR_BASE:
5200 case VMX_VMCS_GUEST_IDTR_BASE:
5201 case VMX_VMCS_GUEST_CS_BASE:
5202 case VMX_VMCS_GUEST_DS_BASE:
5203 case VMX_VMCS_GUEST_ES_BASE:
5204 case VMX_VMCS_GUEST_FS_BASE:
5205 case VMX_VMCS_GUEST_GS_BASE:
5206 case VMX_VMCS_GUEST_SS_BASE:
5207 case VMX_VMCS_GUEST_LDTR_BASE:
5208 case VMX_VMCS_GUEST_TR_BASE:
5209 case VMX_VMCS_GUEST_CR3:
5210 return true;
5211 }
5212 return false;
5213}
5214
5215static bool hmR0VmxIsValidReadField(uint32_t idxField)
5216{
5217 switch (idxField)
5218 {
5219 /* Read-only fields. */
5220 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5221 return true;
5222 }
5223 /* Remaining readable fields should also be writable. */
5224 return hmR0VmxIsValidWriteField(idxField);
5225}
5226#endif /* VBOX_STRICT */
5227
5228
5229/**
5230 * Executes the specified handler in 64-bit mode.
5231 *
5232 * @returns VBox status code (no informational status codes).
5233 * @param pVCpu The cross context virtual CPU structure.
5234 * @param enmOp The operation to perform.
5235 * @param cParams Number of parameters.
5236 * @param paParam Array of 32-bit parameters.
5237 */
5238VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
5239{
5240 PVM pVM = pVCpu->CTX_SUFF(pVM);
5241 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5242 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5243 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5244 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5245
5246#ifdef VBOX_STRICT
5247 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5248 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5249
5250 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5251 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5252#endif
5253
5254 /* Disable interrupts. */
5255 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5256
5257#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5258 RTCPUID idHostCpu = RTMpCpuId();
5259 CPUMR0SetLApic(pVCpu, idHostCpu);
5260#endif
5261
5262 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5263 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5264
5265 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5266 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5267 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5268
5269 /* Leave VMX Root Mode. */
5270 VMXDisable();
5271
5272 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5273
5274 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5275 CPUMSetHyperEIP(pVCpu, enmOp);
5276 for (int i = (int)cParams - 1; i >= 0; i--)
5277 CPUMPushHyper(pVCpu, paParam[i]);
5278
5279 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5280
5281 /* Call the switcher. */
5282 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
5283 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5284
5285 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5286 /* Make sure the VMX instructions don't cause #UD faults. */
5287 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5288
5289 /* Re-enter VMX Root Mode */
5290 int rc2 = VMXEnable(HCPhysCpuPage);
5291 if (RT_FAILURE(rc2))
5292 {
5293 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5294 ASMSetFlags(fOldEFlags);
5295 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5296 return rc2;
5297 }
5298
5299 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5300 AssertRC(rc2);
5301 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5302 Assert(!(ASMGetFlags() & X86_EFL_IF));
5303 ASMSetFlags(fOldEFlags);
5304 return rc;
5305}
5306
5307
5308/**
5309 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5310 * supporting 64-bit guests.
5311 *
5312 * @returns VBox status code.
5313 * @param fResume Whether to VMLAUNCH or VMRESUME.
5314 * @param pCtx Pointer to the guest-CPU context.
5315 * @param pCache Pointer to the VMCS cache.
5316 * @param pVM The cross context VM structure.
5317 * @param pVCpu The cross context virtual CPU structure.
5318 */
5319DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5320{
5321 NOREF(fResume);
5322
5323 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5324 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5325
5326#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5327 pCache->uPos = 1;
5328 pCache->interPD = PGMGetInterPaeCR3(pVM);
5329 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5330#endif
5331
5332#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5333 pCache->TestIn.HCPhysCpuPage = 0;
5334 pCache->TestIn.HCPhysVmcs = 0;
5335 pCache->TestIn.pCache = 0;
5336 pCache->TestOut.HCPhysVmcs = 0;
5337 pCache->TestOut.pCache = 0;
5338 pCache->TestOut.pCtx = 0;
5339 pCache->TestOut.eflags = 0;
5340#else
5341 NOREF(pCache);
5342#endif
5343
5344 uint32_t aParam[10];
5345 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5346 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5347 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5348 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5349 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5350 aParam[5] = 0;
5351 aParam[6] = VM_RC_ADDR(pVM, pVM);
5352 aParam[7] = 0;
5353 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5354 aParam[9] = 0;
5355
5356#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5357 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5358 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5359#endif
5360 int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5361
5362#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5363 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5364 Assert(pCtx->dr[4] == 10);
5365 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5366#endif
5367
5368#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5369 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5370 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5371 pVCpu->hm.s.vmx.HCPhysVmcs));
5372 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5373 pCache->TestOut.HCPhysVmcs));
5374 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5375 pCache->TestOut.pCache));
5376 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5377 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5378 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5379 pCache->TestOut.pCtx));
5380 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5381#endif
5382 NOREF(pCtx);
5383 return rc;
5384}
5385
5386
5387/**
5388 * Initialize the VMCS-Read cache.
5389 *
5390 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5391 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5392 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5393 * (those that have a 32-bit FULL & HIGH part).
5394 *
5395 * @returns VBox status code.
5396 * @param pVCpu The cross context virtual CPU structure.
5397 */
5398static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
5399{
5400#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5401 do { \
5402 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5403 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5404 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5405 ++cReadFields; \
5406 } while (0)
5407
5408 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5409 uint32_t cReadFields = 0;
5410
5411 /*
5412 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5413 * and serve to indicate exceptions to the rules.
5414 */
5415
5416 /* Guest-natural selector base fields. */
5417#if 0
5418 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5419 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5420 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5421#endif
5422 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5423 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5424 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5425 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5426 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5427 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5428 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5429 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5430 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5431 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5432 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5433 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5434#if 0
5435 /* Unused natural width guest-state fields. */
5436 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5437 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5438#endif
5439 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5440 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5441
5442 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5443 these 64-bit fields (using "FULL" and "HIGH" fields). */
5444#if 0
5445 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5446 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5447 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5448 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5449 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5450 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5451 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5452 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5453 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5454#endif
5455
5456 /* Natural width guest-state fields. */
5457 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5458 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_GUEST_LINEAR_ADDR);
5459
5460 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5461 {
5462 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5463 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5464 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5465 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5466 }
5467 else
5468 {
5469 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5470 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5471 }
5472
5473#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5474 return VINF_SUCCESS;
5475}
5476
5477
5478/**
5479 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5480 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5481 * darwin, running 64-bit guests).
5482 *
5483 * @returns VBox status code.
5484 * @param pVCpu The cross context virtual CPU structure.
5485 * @param idxField The VMCS field encoding.
5486 * @param u64Val 16, 32 or 64-bit value.
5487 */
5488VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5489{
5490 int rc;
5491 switch (idxField)
5492 {
5493 /*
5494 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5495 */
5496 /* 64-bit Control fields. */
5497 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5498 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5499 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5500 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5501 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5502 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5503 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5504 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5505 case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
5506 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5507 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5508 case VMX_VMCS64_CTRL_EPTP_FULL:
5509 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5510 /* 64-bit Guest-state fields. */
5511 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5512 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5513 case VMX_VMCS64_GUEST_PAT_FULL:
5514 case VMX_VMCS64_GUEST_EFER_FULL:
5515 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5516 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5517 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5518 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5519 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5520 /* 64-bit Host-state fields. */
5521 case VMX_VMCS64_HOST_PAT_FULL:
5522 case VMX_VMCS64_HOST_EFER_FULL:
5523 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5524 {
5525 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5526 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5527 break;
5528 }
5529
5530 /*
5531 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5532 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5533 */
5534 /* Natural-width Guest-state fields. */
5535 case VMX_VMCS_GUEST_CR3:
5536 case VMX_VMCS_GUEST_ES_BASE:
5537 case VMX_VMCS_GUEST_CS_BASE:
5538 case VMX_VMCS_GUEST_SS_BASE:
5539 case VMX_VMCS_GUEST_DS_BASE:
5540 case VMX_VMCS_GUEST_FS_BASE:
5541 case VMX_VMCS_GUEST_GS_BASE:
5542 case VMX_VMCS_GUEST_LDTR_BASE:
5543 case VMX_VMCS_GUEST_TR_BASE:
5544 case VMX_VMCS_GUEST_GDTR_BASE:
5545 case VMX_VMCS_GUEST_IDTR_BASE:
5546 case VMX_VMCS_GUEST_RSP:
5547 case VMX_VMCS_GUEST_RIP:
5548 case VMX_VMCS_GUEST_SYSENTER_ESP:
5549 case VMX_VMCS_GUEST_SYSENTER_EIP:
5550 {
5551 if (!(RT_HI_U32(u64Val)))
5552 {
5553 /* If this field is 64-bit, VT-x will zero out the top bits. */
5554 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5555 }
5556 else
5557 {
5558 /* Assert that only the 32->64 switcher case should ever come here. */
5559 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5560 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5561 }
5562 break;
5563 }
5564
5565 default:
5566 {
5567 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5568 rc = VERR_INVALID_PARAMETER;
5569 break;
5570 }
5571 }
5572 AssertRCReturn(rc, rc);
5573 return rc;
5574}
5575
5576
5577/**
5578 * Queue up a VMWRITE by using the VMCS write cache.
5579 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5580 *
5581 * @param pVCpu The cross context virtual CPU structure.
5582 * @param idxField The VMCS field encoding.
5583 * @param u64Val 16, 32 or 64-bit value.
5584 */
5585VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5586{
5587 AssertPtr(pVCpu);
5588 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5589
5590 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5591 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5592
5593 /* Make sure there are no duplicates. */
5594 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5595 {
5596 if (pCache->Write.aField[i] == idxField)
5597 {
5598 pCache->Write.aFieldVal[i] = u64Val;
5599 return VINF_SUCCESS;
5600 }
5601 }
5602
5603 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5604 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5605 pCache->Write.cValidEntries++;
5606 return VINF_SUCCESS;
5607}
5608#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5609
5610
5611/**
5612 * Sets up the usage of TSC-offsetting and updates the VMCS.
5613 *
5614 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5615 * VMX preemption timer.
5616 *
5617 * @returns VBox status code.
5618 * @param pVCpu The cross context virtual CPU structure.
5619 *
5620 * @remarks No-long-jump zone!!!
5621 */
5622static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5623{
5624 bool fOffsettedTsc;
5625 bool fParavirtTsc;
5626 PVM pVM = pVCpu->CTX_SUFF(pVM);
5627 uint64_t uTscOffset;
5628 if (pVM->hm.s.vmx.fUsePreemptTimer)
5629 {
5630 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
5631
5632 /* Make sure the returned values have sane upper and lower boundaries. */
5633 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5634 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5635 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5636 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5637
5638 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5639 int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
5640 AssertRC(rc);
5641 }
5642 else
5643 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
5644
5645 if (fParavirtTsc)
5646 {
5647 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5648 information before every VM-entry, hence disable it for performance sake. */
5649#if 0
5650 int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5651 AssertRC(rc);
5652#endif
5653 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5654 }
5655
5656 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
5657 if ( fOffsettedTsc
5658 && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5659 {
5660 if (pVCpu->hm.s.vmx.u64TscOffset != uTscOffset)
5661 {
5662 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
5663 AssertRC(rc);
5664 pVCpu->hm.s.vmx.u64TscOffset = uTscOffset;
5665 }
5666
5667 if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
5668 {
5669 uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
5670 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5671 AssertRC(rc);
5672 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5673 }
5674 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5675 }
5676 else
5677 {
5678 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5679 if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
5680 {
5681 uProcCtls |= VMX_PROC_CTLS_RDTSC_EXIT;
5682 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5683 AssertRC(rc);
5684 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5685 }
5686 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5687 }
5688}
5689
5690
5691/**
5692 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5693 * VM-exit interruption info type.
5694 *
5695 * @returns The IEM exception flags.
5696 * @param uVector The event vector.
5697 * @param uVmxVectorType The VMX event type.
5698 *
5699 * @remarks This function currently only constructs flags required for
5700 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5701 * and CR2 aspects of an exception are not included).
5702 */
5703static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5704{
5705 uint32_t fIemXcptFlags;
5706 switch (uVmxVectorType)
5707 {
5708 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5709 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5710 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5711 break;
5712
5713 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5714 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5715 break;
5716
5717 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5718 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5719 break;
5720
5721 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5722 {
5723 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5724 if (uVector == X86_XCPT_BP)
5725 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5726 else if (uVector == X86_XCPT_OF)
5727 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5728 else
5729 {
5730 fIemXcptFlags = 0;
5731 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5732 }
5733 break;
5734 }
5735
5736 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5737 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5738 break;
5739
5740 default:
5741 fIemXcptFlags = 0;
5742 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5743 break;
5744 }
5745 return fIemXcptFlags;
5746}
5747
5748
5749/**
5750 * Sets an event as a pending event to be injected into the guest.
5751 *
5752 * @param pVCpu The cross context virtual CPU structure.
5753 * @param u32IntInfo The VM-entry interruption-information field.
5754 * @param cbInstr The VM-entry instruction length in bytes (for software
5755 * interrupts, exceptions and privileged software
5756 * exceptions).
5757 * @param u32ErrCode The VM-entry exception error code.
5758 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5759 * page-fault.
5760 *
5761 * @remarks Statistics counter assumes this is a guest event being injected or
5762 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5763 * always incremented.
5764 */
5765DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5766 RTGCUINTPTR GCPtrFaultAddress)
5767{
5768 Assert(!pVCpu->hm.s.Event.fPending);
5769 pVCpu->hm.s.Event.fPending = true;
5770 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5771 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5772 pVCpu->hm.s.Event.cbInstr = cbInstr;
5773 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5774}
5775
5776
5777/**
5778 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5779 *
5780 * @param pVCpu The cross context virtual CPU structure.
5781 */
5782DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
5783{
5784 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
5785 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5786 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5787 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5788 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5789}
5790
5791
5792/**
5793 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
5794 *
5795 * @param pVCpu The cross context virtual CPU structure.
5796 */
5797DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
5798{
5799 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
5800 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5801 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
5802 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5803 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5804}
5805
5806
5807/**
5808 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
5809 *
5810 * @param pVCpu The cross context virtual CPU structure.
5811 */
5812DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
5813{
5814 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
5815 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5816 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
5817 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5818 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5819}
5820
5821
5822#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5823/**
5824 * Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
5825 *
5826 * @param pVCpu The cross context virtual CPU structure.
5827 * @param u32ErrCode The error code for the general-protection exception.
5828 */
5829DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
5830{
5831 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
5832 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5833 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5834 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5835 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrCode, 0 /* GCPtrFaultAddress */);
5836}
5837
5838
5839/**
5840 * Sets a stack (\#SS) exception as pending-for-injection into the VM.
5841 *
5842 * @param pVCpu The cross context virtual CPU structure.
5843 * @param u32ErrCode The error code for the stack exception.
5844 */
5845DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
5846{
5847 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
5848 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5849 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5850 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5851 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrCode, 0 /* GCPtrFaultAddress */);
5852}
5853
5854
5855/**
5856 * Decodes the memory operand of an instruction that caused a VM-exit.
5857 *
5858 * The VM-exit qualification field provides the displacement field for memory
5859 * operand instructions, if any.
5860 *
5861 * @returns Strict VBox status code (i.e. informational status codes too).
5862 * @retval VINF_SUCCESS if the operand was successfully decoded.
5863 * @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
5864 * operand.
5865 * @param pVCpu The cross context virtual CPU structure.
5866 * @param uExitInstrInfo The VM-exit instruction information field.
5867 * @param enmMemAccess The memory operand's access type (read or write).
5868 * @param GCPtrDisp The instruction displacement field, if any. For
5869 * RIP-relative addressing pass RIP + displacement here.
5870 * @param pGCPtrMem Where to store the effective destination memory address.
5871 */
5872static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
5873 PRTGCPTR pGCPtrMem)
5874{
5875 Assert(pGCPtrMem);
5876 Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
5877 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_EFER | CPUMCTX_EXTRN_CR0);
5878
5879 static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
5880 static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
5881 AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
5882
5883 VMXEXITINSTRINFO ExitInstrInfo;
5884 ExitInstrInfo.u = uExitInstrInfo;
5885 uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
5886 uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
5887 bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
5888 uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
5889 uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
5890 bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
5891 uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
5892 bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
5893 bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
5894
5895 /*
5896 * Validate instruction information.
5897 * This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
5898 */
5899 AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
5900 ("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
5901 AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
5902 ("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
5903 AssertLogRelMsgReturn(fIsMemOperand,
5904 ("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
5905
5906 /*
5907 * Compute the complete effective address.
5908 *
5909 * See AMD instruction spec. 1.4.2 "SIB Byte Format"
5910 * See AMD spec. 4.5.2 "Segment Registers".
5911 */
5912 RTGCPTR GCPtrMem = GCPtrDisp;
5913 if (fBaseRegValid)
5914 GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
5915 if (fIdxRegValid)
5916 GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
5917
5918 RTGCPTR const GCPtrOff = GCPtrMem;
5919 if ( !fIsLongMode
5920 || iSegReg >= X86_SREG_FS)
5921 GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
5922 GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
5923
5924 /*
5925 * Validate effective address.
5926 * See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
5927 */
5928 uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
5929 Assert(cbAccess > 0);
5930 if (fIsLongMode)
5931 {
5932 if (X86_IS_CANONICAL(GCPtrMem))
5933 {
5934 *pGCPtrMem = GCPtrMem;
5935 return VINF_SUCCESS;
5936 }
5937
5938 /** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
5939 * "Data Limit Checks in 64-bit Mode". */
5940 Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
5941 hmR0VmxSetPendingXcptGP(pVCpu, 0);
5942 return VINF_HM_PENDING_XCPT;
5943 }
5944
5945 /*
5946 * This is a watered down version of iemMemApplySegment().
5947 * Parts that are not applicable for VMX instructions like real-or-v8086 mode
5948 * and segment CPL/DPL checks are skipped.
5949 */
5950 RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
5951 RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
5952 PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
5953
5954 /* Check if the segment is present and usable. */
5955 if ( pSel->Attr.n.u1Present
5956 && !pSel->Attr.n.u1Unusable)
5957 {
5958 Assert(pSel->Attr.n.u1DescType);
5959 if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
5960 {
5961 /* Check permissions for the data segment. */
5962 if ( enmMemAccess == VMXMEMACCESS_WRITE
5963 && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
5964 {
5965 Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
5966 hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
5967 return VINF_HM_PENDING_XCPT;
5968 }
5969
5970 /* Check limits if it's a normal data segment. */
5971 if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
5972 {
5973 if ( GCPtrFirst32 > pSel->u32Limit
5974 || GCPtrLast32 > pSel->u32Limit)
5975 {
5976 Log4Func(("Data segment limit exceeded."
5977 "iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
5978 GCPtrLast32, pSel->u32Limit));
5979 if (iSegReg == X86_SREG_SS)
5980 hmR0VmxSetPendingXcptSS(pVCpu, 0);
5981 else
5982 hmR0VmxSetPendingXcptGP(pVCpu, 0);
5983 return VINF_HM_PENDING_XCPT;
5984 }
5985 }
5986 else
5987 {
5988 /* Check limits if it's an expand-down data segment.
5989 Note! The upper boundary is defined by the B bit, not the G bit! */
5990 if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
5991 || GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
5992 {
5993 Log4Func(("Expand-down data segment limit exceeded."
5994 "iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
5995 GCPtrLast32, pSel->u32Limit));
5996 if (iSegReg == X86_SREG_SS)
5997 hmR0VmxSetPendingXcptSS(pVCpu, 0);
5998 else
5999 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6000 return VINF_HM_PENDING_XCPT;
6001 }
6002 }
6003 }
6004 else
6005 {
6006 /* Check permissions for the code segment. */
6007 if ( enmMemAccess == VMXMEMACCESS_WRITE
6008 || ( enmMemAccess == VMXMEMACCESS_READ
6009 && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
6010 {
6011 Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
6012 Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
6013 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6014 return VINF_HM_PENDING_XCPT;
6015 }
6016
6017 /* Check limits for the code segment (normal/expand-down not applicable for code segments). */
6018 if ( GCPtrFirst32 > pSel->u32Limit
6019 || GCPtrLast32 > pSel->u32Limit)
6020 {
6021 Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
6022 GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
6023 if (iSegReg == X86_SREG_SS)
6024 hmR0VmxSetPendingXcptSS(pVCpu, 0);
6025 else
6026 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6027 return VINF_HM_PENDING_XCPT;
6028 }
6029 }
6030 }
6031 else
6032 {
6033 Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6034 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6035 return VINF_HM_PENDING_XCPT;
6036 }
6037
6038 *pGCPtrMem = GCPtrMem;
6039 return VINF_SUCCESS;
6040}
6041
6042
6043/**
6044 * Perform the relevant VMX instruction checks for VM-exits that occurred due to the
6045 * guest attempting to execute a VMX instruction.
6046 *
6047 * @returns Strict VBox status code (i.e. informational status codes too).
6048 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
6049 * @retval VINF_HM_PENDING_XCPT if an exception was raised.
6050 *
6051 * @param pVCpu The cross context virtual CPU structure.
6052 * @param uExitReason The VM-exit reason.
6053 *
6054 * @todo NstVmx: Document other error codes when VM-exit is implemented.
6055 * @remarks No-long-jump zone!!!
6056 */
6057static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
6058{
6059 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS
6060 | CPUMCTX_EXTRN_HWVIRT);
6061
6062 if ( CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx)
6063 || ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6064 && !CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx)))
6065 {
6066 Log4Func(("In real/v86-mode or long-mode outside 64-bit code segment -> #UD\n"));
6067 hmR0VmxSetPendingXcptUD(pVCpu);
6068 return VINF_HM_PENDING_XCPT;
6069 }
6070
6071 if (uExitReason == VMX_EXIT_VMXON)
6072 {
6073 /*
6074 * We check CR4.VMXE because it is required to be always set while in VMX operation
6075 * by physical CPUs and our CR4 read shadow is only consulted when executing specific
6076 * instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
6077 * otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
6078 */
6079 if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
6080 {
6081 Log4Func(("CR4.VMXE is not set -> #UD\n"));
6082 hmR0VmxSetPendingXcptUD(pVCpu);
6083 return VINF_HM_PENDING_XCPT;
6084 }
6085 }
6086 else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
6087 {
6088 /*
6089 * The guest has not entered VMX operation but attempted to execute a VMX instruction
6090 * (other than VMXON), we need to raise a #UD.
6091 */
6092 Log4Func(("Not in VMX root mode -> #UD\n"));
6093 hmR0VmxSetPendingXcptUD(pVCpu);
6094 return VINF_HM_PENDING_XCPT;
6095 }
6096
6097 if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
6098 {
6099 /*
6100 * The nested-guest attempted to execute a VMX instruction, cause a VM-exit and let
6101 * the guest hypervisor deal with it.
6102 */
6103 /** @todo NSTVMX: Trigger a VM-exit */
6104 }
6105
6106 /*
6107 * VMX instructions require CPL 0 except in VMX non-root mode where the VM-exit intercept
6108 * (above) takes preceedence over the CPL check.
6109 */
6110 if (CPUMGetGuestCPL(pVCpu) > 0)
6111 {
6112 Log4Func(("CPL > 0 -> #GP(0)\n"));
6113 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6114 return VINF_HM_PENDING_XCPT;
6115 }
6116
6117 return VINF_SUCCESS;
6118}
6119
6120#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
6121
6122
6123/**
6124 * Handle a condition that occurred while delivering an event through the guest
6125 * IDT.
6126 *
6127 * @returns Strict VBox status code (i.e. informational status codes too).
6128 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
6129 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
6130 * to continue execution of the guest which will delivery the \#DF.
6131 * @retval VINF_EM_RESET if we detected a triple-fault condition.
6132 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
6133 *
6134 * @param pVCpu The cross context virtual CPU structure.
6135 * @param pVmxTransient Pointer to the VMX transient structure.
6136 *
6137 * @remarks No-long-jump zone!!!
6138 */
6139static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6140{
6141 uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
6142
6143 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
6144 rc2 |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
6145 AssertRCReturn(rc2, rc2);
6146
6147 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
6148 if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6149 {
6150 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
6151 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
6152
6153 /*
6154 * If the event was a software interrupt (generated with INT n) or a software exception
6155 * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
6156 * can handle the VM-exit and continue guest execution which will re-execute the
6157 * instruction rather than re-injecting the exception, as that can cause premature
6158 * trips to ring-3 before injection and involve TRPM which currently has no way of
6159 * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
6160 * the problem).
6161 */
6162 IEMXCPTRAISE enmRaise;
6163 IEMXCPTRAISEINFO fRaiseInfo;
6164 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6165 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6166 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6167 {
6168 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
6169 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6170 }
6171 else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
6172 {
6173 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
6174 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
6175 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
6176 /** @todo Make AssertMsgReturn as just AssertMsg later. */
6177 AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
6178 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
6179 uExitVectorType), VERR_VMX_IPE_5);
6180
6181 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
6182
6183 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
6184 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
6185 {
6186 pVmxTransient->fVectoringPF = true;
6187 enmRaise = IEMXCPTRAISE_PREV_EVENT;
6188 }
6189 }
6190 else
6191 {
6192 /*
6193 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
6194 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
6195 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
6196 */
6197 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6198 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6199 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
6200 enmRaise = IEMXCPTRAISE_PREV_EVENT;
6201 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6202 }
6203
6204 /*
6205 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
6206 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
6207 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
6208 * subsequent VM-entry would fail.
6209 *
6210 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
6211 */
6212 if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS)
6213 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6214 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
6215 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
6216 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
6217 {
6218 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6219 }
6220
6221 switch (enmRaise)
6222 {
6223 case IEMXCPTRAISE_CURRENT_XCPT:
6224 {
6225 Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
6226 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
6227 Assert(rcStrict == VINF_SUCCESS);
6228 break;
6229 }
6230
6231 case IEMXCPTRAISE_PREV_EVENT:
6232 {
6233 uint32_t u32ErrCode;
6234 if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
6235 {
6236 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6237 AssertRCReturn(rc2, rc2);
6238 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6239 }
6240 else
6241 u32ErrCode = 0;
6242
6243 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
6244 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6245 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6246 0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
6247
6248 Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
6249 pVCpu->hm.s.Event.u32ErrCode));
6250 Assert(rcStrict == VINF_SUCCESS);
6251 break;
6252 }
6253
6254 case IEMXCPTRAISE_REEXEC_INSTR:
6255 Assert(rcStrict == VINF_SUCCESS);
6256 break;
6257
6258 case IEMXCPTRAISE_DOUBLE_FAULT:
6259 {
6260 /*
6261 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
6262 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
6263 */
6264 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6265 {
6266 pVmxTransient->fVectoringDoublePF = true;
6267 Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
6268 pVCpu->cpum.GstCtx.cr2));
6269 rcStrict = VINF_SUCCESS;
6270 }
6271 else
6272 {
6273 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6274 hmR0VmxSetPendingXcptDF(pVCpu);
6275 Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
6276 uIdtVector, uExitVector));
6277 rcStrict = VINF_HM_DOUBLE_FAULT;
6278 }
6279 break;
6280 }
6281
6282 case IEMXCPTRAISE_TRIPLE_FAULT:
6283 {
6284 Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
6285 rcStrict = VINF_EM_RESET;
6286 break;
6287 }
6288
6289 case IEMXCPTRAISE_CPU_HANG:
6290 {
6291 Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
6292 rcStrict = VERR_EM_GUEST_CPU_HANG;
6293 break;
6294 }
6295
6296 default:
6297 {
6298 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6299 rcStrict = VERR_VMX_IPE_2;
6300 break;
6301 }
6302 }
6303 }
6304 else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6305 && VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6306 && uExitVector != X86_XCPT_DF
6307 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
6308 {
6309 /*
6310 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6311 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6312 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6313 */
6314 if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
6315 {
6316 Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
6317 VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6318 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6319 }
6320 }
6321
6322 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
6323 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
6324 return rcStrict;
6325}
6326
6327
6328/**
6329 * Imports a guest segment register from the current VMCS into
6330 * the guest-CPU context.
6331 *
6332 * @returns VBox status code.
6333 * @param pVCpu The cross context virtual CPU structure.
6334 * @param idxSel Index of the selector in the VMCS.
6335 * @param idxLimit Index of the segment limit in the VMCS.
6336 * @param idxBase Index of the segment base in the VMCS.
6337 * @param idxAccess Index of the access rights of the segment in the VMCS.
6338 * @param pSelReg Pointer to the segment selector.
6339 *
6340 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6341 * do not log!
6342 *
6343 * @remarks Never call this function directly!!! Use the
6344 * HMVMX_IMPORT_SREG() macro as that takes care
6345 * of whether to read from the VMCS cache or not.
6346 */
6347static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6348 PCPUMSELREG pSelReg)
6349{
6350 NOREF(pVCpu);
6351
6352 uint32_t u32Sel;
6353 uint32_t u32Limit;
6354 uint32_t u32Attr;
6355 uint64_t u64Base;
6356 int rc = VMXReadVmcs32(idxSel, &u32Sel);
6357 rc |= VMXReadVmcs32(idxLimit, &u32Limit);
6358 rc |= VMXReadVmcs32(idxAccess, &u32Attr);
6359 rc |= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
6360 AssertRCReturn(rc, rc);
6361
6362 pSelReg->Sel = (uint16_t)u32Sel;
6363 pSelReg->ValidSel = (uint16_t)u32Sel;
6364 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6365 pSelReg->u32Limit = u32Limit;
6366 pSelReg->u64Base = u64Base;
6367 pSelReg->Attr.u = u32Attr;
6368
6369 /*
6370 * If VT-x marks the segment as unusable, most other bits remain undefined:
6371 * - For CS the L, D and G bits have meaning.
6372 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6373 * - For the remaining data segments no bits are defined.
6374 *
6375 * The present bit and the unusable bit has been observed to be set at the
6376 * same time (the selector was supposed to be invalid as we started executing
6377 * a V8086 interrupt in ring-0).
6378 *
6379 * What should be important for the rest of the VBox code, is that the P bit is
6380 * cleared. Some of the other VBox code recognizes the unusable bit, but
6381 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6382 * safe side here, we'll strip off P and other bits we don't care about. If
6383 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6384 *
6385 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6386 */
6387 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6388 {
6389 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6390
6391 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6392 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6393 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6394#ifdef VBOX_STRICT
6395 VMMRZCallRing3Disable(pVCpu);
6396 Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
6397# ifdef DEBUG_bird
6398 AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
6399 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6400 idxSel, u32Sel, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6401# endif
6402 VMMRZCallRing3Enable(pVCpu);
6403#endif
6404 }
6405 return VINF_SUCCESS;
6406}
6407
6408
6409/**
6410 * Imports the guest RIP from the VMCS back into the guest-CPU context.
6411 *
6412 * @returns VBox status code.
6413 * @param pVCpu The cross context virtual CPU structure.
6414 *
6415 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6416 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6417 * instead!!!
6418 */
6419DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6420{
6421 uint64_t u64Val;
6422 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6423 if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6424 {
6425 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6426 if (RT_SUCCESS(rc))
6427 {
6428 pCtx->rip = u64Val;
6429 EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
6430 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6431 }
6432 return rc;
6433 }
6434 return VINF_SUCCESS;
6435}
6436
6437
6438/**
6439 * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6440 *
6441 * @returns VBox status code.
6442 * @param pVCpu The cross context virtual CPU structure.
6443 *
6444 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6445 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6446 * instead!!!
6447 */
6448DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6449{
6450 uint32_t u32Val;
6451 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6452 if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6453 {
6454 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6455 if (RT_SUCCESS(rc))
6456 {
6457 pCtx->eflags.u32 = u32Val;
6458
6459 /* Restore eflags for real-on-v86-mode hack. */
6460 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6461 {
6462 pCtx->eflags.Bits.u1VM = 0;
6463 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6464 }
6465 }
6466 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6467 return rc;
6468 }
6469 return VINF_SUCCESS;
6470}
6471
6472
6473/**
6474 * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6475 * context.
6476 *
6477 * @returns VBox status code.
6478 * @param pVCpu The cross context virtual CPU structure.
6479 *
6480 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6481 * do not log!
6482 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6483 * instead!!!
6484 */
6485DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6486{
6487 uint32_t u32Val;
6488 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6489 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
6490 if (RT_SUCCESS(rc))
6491 {
6492 /*
6493 * We additionally have a requirement to import RIP, RFLAGS depending on whether we
6494 * might need them in hmR0VmxEvaluatePendingEvent().
6495 */
6496 if (!u32Val)
6497 {
6498 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6499 {
6500 rc = hmR0VmxImportGuestRip(pVCpu);
6501 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6502 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6503 }
6504
6505 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
6506 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6507 }
6508 else
6509 {
6510 rc = hmR0VmxImportGuestRip(pVCpu);
6511 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6512
6513 if (u32Val & ( VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
6514 | VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
6515 {
6516 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6517 }
6518 else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6519 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6520
6521 if (u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
6522 {
6523 if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
6524 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6525 }
6526 else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
6527 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6528 }
6529 }
6530 return rc;
6531}
6532
6533
6534/**
6535 * Worker for VMXR0ImportStateOnDemand.
6536 *
6537 * @returns VBox status code.
6538 * @param pVCpu The cross context virtual CPU structure.
6539 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6540 */
6541static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6542{
6543#define VMXLOCAL_BREAK_RC(a_rc) \
6544 if (RT_FAILURE(a_rc)) \
6545 break
6546
6547 int rc = VINF_SUCCESS;
6548 PVM pVM = pVCpu->CTX_SUFF(pVM);
6549 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6550 uint64_t u64Val;
6551 uint32_t u32Val;
6552
6553 Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6554 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6555
6556 /*
6557 * We disable interrupts to make the updating of the state and in particular
6558 * the fExtrn modification atomic wrt to preemption hooks.
6559 */
6560 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6561
6562 fWhat &= pCtx->fExtrn;
6563 if (fWhat)
6564 {
6565 do
6566 {
6567 if (fWhat & CPUMCTX_EXTRN_RIP)
6568 {
6569 rc = hmR0VmxImportGuestRip(pVCpu);
6570 VMXLOCAL_BREAK_RC(rc);
6571 }
6572
6573 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6574 {
6575 rc = hmR0VmxImportGuestRFlags(pVCpu);
6576 VMXLOCAL_BREAK_RC(rc);
6577 }
6578
6579 if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6580 {
6581 rc = hmR0VmxImportGuestIntrState(pVCpu);
6582 VMXLOCAL_BREAK_RC(rc);
6583 }
6584
6585 if (fWhat & CPUMCTX_EXTRN_RSP)
6586 {
6587 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6588 VMXLOCAL_BREAK_RC(rc);
6589 pCtx->rsp = u64Val;
6590 }
6591
6592 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6593 {
6594 if (fWhat & CPUMCTX_EXTRN_CS)
6595 {
6596 rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6597 rc |= hmR0VmxImportGuestRip(pVCpu);
6598 VMXLOCAL_BREAK_RC(rc);
6599 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6600 pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6601 EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true);
6602 }
6603 if (fWhat & CPUMCTX_EXTRN_SS)
6604 {
6605 rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6606 VMXLOCAL_BREAK_RC(rc);
6607 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6608 pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6609 }
6610 if (fWhat & CPUMCTX_EXTRN_DS)
6611 {
6612 rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6613 VMXLOCAL_BREAK_RC(rc);
6614 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6615 pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6616 }
6617 if (fWhat & CPUMCTX_EXTRN_ES)
6618 {
6619 rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6620 VMXLOCAL_BREAK_RC(rc);
6621 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6622 pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6623 }
6624 if (fWhat & CPUMCTX_EXTRN_FS)
6625 {
6626 rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6627 VMXLOCAL_BREAK_RC(rc);
6628 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6629 pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6630 }
6631 if (fWhat & CPUMCTX_EXTRN_GS)
6632 {
6633 rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6634 VMXLOCAL_BREAK_RC(rc);
6635 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6636 pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6637 }
6638 }
6639
6640 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6641 {
6642 if (fWhat & CPUMCTX_EXTRN_LDTR)
6643 {
6644 rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6645 VMXLOCAL_BREAK_RC(rc);
6646 }
6647
6648 if (fWhat & CPUMCTX_EXTRN_GDTR)
6649 {
6650 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6651 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6652 VMXLOCAL_BREAK_RC(rc);
6653 pCtx->gdtr.pGdt = u64Val;
6654 pCtx->gdtr.cbGdt = u32Val;
6655 }
6656
6657 /* Guest IDTR. */
6658 if (fWhat & CPUMCTX_EXTRN_IDTR)
6659 {
6660 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6661 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6662 VMXLOCAL_BREAK_RC(rc);
6663 pCtx->idtr.pIdt = u64Val;
6664 pCtx->idtr.cbIdt = u32Val;
6665 }
6666
6667 /* Guest TR. */
6668 if (fWhat & CPUMCTX_EXTRN_TR)
6669 {
6670 /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6671 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6672 {
6673 rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6674 VMXLOCAL_BREAK_RC(rc);
6675 }
6676 }
6677 }
6678
6679 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6680 {
6681 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6682 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6683 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6684 pCtx->SysEnter.cs = u32Val;
6685 VMXLOCAL_BREAK_RC(rc);
6686 }
6687
6688#if HC_ARCH_BITS == 64
6689 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6690 {
6691 if ( pVM->hm.s.fAllow64BitGuests
6692 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6693 pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6694 }
6695
6696 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6697 {
6698 if ( pVM->hm.s.fAllow64BitGuests
6699 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6700 {
6701 pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6702 pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6703 pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6704 }
6705 }
6706#endif
6707
6708 if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
6709#if HC_ARCH_BITS == 32
6710 || (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS))
6711#endif
6712 )
6713 {
6714 PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6715 uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6716 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6717 {
6718 switch (pMsr->u32Msr)
6719 {
6720#if HC_ARCH_BITS == 32
6721 case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6722 case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6723 case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6724 case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6725#endif
6726 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6727 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6728 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit */ break;
6729 default:
6730 {
6731 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6732 ASMSetFlags(fEFlags);
6733 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6734 cMsrs));
6735 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6736 }
6737 }
6738 }
6739 }
6740
6741 if (fWhat & CPUMCTX_EXTRN_DR7)
6742 {
6743 if (!pVCpu->hm.s.fUsingHyperDR7)
6744 {
6745 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6746 rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6747 VMXLOCAL_BREAK_RC(rc);
6748 pCtx->dr[7] = u32Val;
6749 }
6750 }
6751
6752 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6753 {
6754 uint32_t u32Shadow;
6755 if (fWhat & CPUMCTX_EXTRN_CR0)
6756 {
6757 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6758 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6759 VMXLOCAL_BREAK_RC(rc);
6760 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr0Mask)
6761 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr0Mask);
6762 VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6763 CPUMSetGuestCR0(pVCpu, u32Val);
6764 VMMRZCallRing3Enable(pVCpu);
6765 }
6766
6767 if (fWhat & CPUMCTX_EXTRN_CR4)
6768 {
6769 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6770 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6771 VMXLOCAL_BREAK_RC(rc);
6772 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr4Mask)
6773 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr4Mask);
6774 CPUMSetGuestCR4(pVCpu, u32Val);
6775 }
6776
6777 if (fWhat & CPUMCTX_EXTRN_CR3)
6778 {
6779 /* CR0.PG bit changes are always intercepted, so it's up to date. */
6780 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6781 || ( pVM->hm.s.fNestedPaging
6782 && CPUMIsGuestPagingEnabledEx(pCtx)))
6783 {
6784 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6785 if (pCtx->cr3 != u64Val)
6786 {
6787 CPUMSetGuestCR3(pVCpu, u64Val);
6788 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6789 }
6790
6791 /* If the guest is in PAE mode, sync back the PDPE's into the guest state.
6792 Note: CR4.PAE, CR0.PG, EFER bit changes are always intercepted, so they're up to date. */
6793 if (CPUMIsGuestInPAEModeEx(pCtx))
6794 {
6795 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6796 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6797 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6798 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6799 VMXLOCAL_BREAK_RC(rc);
6800 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6801 }
6802 }
6803 }
6804 }
6805 } while (0);
6806
6807 if (RT_SUCCESS(rc))
6808 {
6809 /* Update fExtrn. */
6810 pCtx->fExtrn &= ~fWhat;
6811
6812 /* If everything has been imported, clear the HM keeper bit. */
6813 if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6814 {
6815 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6816 Assert(!pCtx->fExtrn);
6817 }
6818 }
6819 }
6820 else
6821 AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6822
6823 ASMSetFlags(fEFlags);
6824
6825 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6826
6827 /*
6828 * Honor any pending CR3 updates.
6829 *
6830 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6831 * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6832 * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6833 *
6834 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6835 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6836 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6837 * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6838 *
6839 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6840 */
6841 if (VMMRZCallRing3IsEnabled(pVCpu))
6842 {
6843 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6844 {
6845 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
6846 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6847 }
6848
6849 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6850 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6851
6852 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6853 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6854 }
6855
6856 return VINF_SUCCESS;
6857#undef VMXLOCAL_BREAK_RC
6858}
6859
6860
6861/**
6862 * Saves the guest state from the VMCS into the guest-CPU context.
6863 *
6864 * @returns VBox status code.
6865 * @param pVCpu The cross context virtual CPU structure.
6866 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6867 */
6868VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6869{
6870 return hmR0VmxImportGuestState(pVCpu, fWhat);
6871}
6872
6873
6874/**
6875 * Check per-VM and per-VCPU force flag actions that require us to go back to
6876 * ring-3 for one reason or another.
6877 *
6878 * @returns Strict VBox status code (i.e. informational status codes too)
6879 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6880 * ring-3.
6881 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6882 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6883 * interrupts)
6884 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6885 * all EMTs to be in ring-3.
6886 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6887 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6888 * to the EM loop.
6889 *
6890 * @param pVCpu The cross context virtual CPU structure.
6891 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6892 */
6893static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
6894{
6895 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6896
6897 /*
6898 * Anything pending? Should be more likely than not if we're doing a good job.
6899 */
6900 PVM pVM = pVCpu->CTX_SUFF(pVM);
6901 if ( !fStepping
6902 ? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6903 && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6904 : !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6905 && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6906 return VINF_SUCCESS;
6907
6908 /* Pending PGM C3 sync. */
6909 if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6910 {
6911 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6912 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
6913 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
6914 VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6915 if (rcStrict2 != VINF_SUCCESS)
6916 {
6917 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6918 Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6919 return rcStrict2;
6920 }
6921 }
6922
6923 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6924 if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
6925 || VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6926 {
6927 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6928 int rc2 = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
6929 Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6930 return rc2;
6931 }
6932
6933 /* Pending VM request packets, such as hardware interrupts. */
6934 if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
6935 || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
6936 {
6937 Log4Func(("Pending VM request forcing us back to ring-3\n"));
6938 return VINF_EM_PENDING_REQUEST;
6939 }
6940
6941 /* Pending PGM pool flushes. */
6942 if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6943 {
6944 Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6945 return VINF_PGM_POOL_FLUSH_PENDING;
6946 }
6947
6948 /* Pending DMA requests. */
6949 if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
6950 {
6951 Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6952 return VINF_EM_RAW_TO_R3;
6953 }
6954
6955 return VINF_SUCCESS;
6956}
6957
6958
6959/**
6960 * Converts any TRPM trap into a pending HM event. This is typically used when
6961 * entering from ring-3 (not longjmp returns).
6962 *
6963 * @param pVCpu The cross context virtual CPU structure.
6964 */
6965static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6966{
6967 Assert(TRPMHasTrap(pVCpu));
6968 Assert(!pVCpu->hm.s.Event.fPending);
6969
6970 uint8_t uVector;
6971 TRPMEVENT enmTrpmEvent;
6972 RTGCUINT uErrCode;
6973 RTGCUINTPTR GCPtrFaultAddress;
6974 uint8_t cbInstr;
6975
6976 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6977 AssertRC(rc);
6978
6979 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6980 uint32_t u32IntInfo = uVector | VMX_EXIT_INT_INFO_VALID;
6981 if (enmTrpmEvent == TRPM_TRAP)
6982 {
6983 switch (uVector)
6984 {
6985 case X86_XCPT_NMI:
6986 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6987 break;
6988
6989 case X86_XCPT_BP:
6990 case X86_XCPT_OF:
6991 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6992 break;
6993
6994 case X86_XCPT_PF:
6995 case X86_XCPT_DF:
6996 case X86_XCPT_TS:
6997 case X86_XCPT_NP:
6998 case X86_XCPT_SS:
6999 case X86_XCPT_GP:
7000 case X86_XCPT_AC:
7001 u32IntInfo |= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7002 RT_FALL_THRU();
7003 default:
7004 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7005 break;
7006 }
7007 }
7008 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7009 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7010 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7011 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7012 else
7013 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7014
7015 rc = TRPMResetTrap(pVCpu);
7016 AssertRC(rc);
7017 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7018 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7019
7020 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7021}
7022
7023
7024/**
7025 * Converts the pending HM event into a TRPM trap.
7026 *
7027 * @param pVCpu The cross context virtual CPU structure.
7028 */
7029static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7030{
7031 Assert(pVCpu->hm.s.Event.fPending);
7032
7033 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7034 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7035 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
7036 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7037
7038 /* If a trap was already pending, we did something wrong! */
7039 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7040
7041 TRPMEVENT enmTrapType;
7042 switch (uVectorType)
7043 {
7044 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7045 enmTrapType = TRPM_HARDWARE_INT;
7046 break;
7047
7048 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7049 enmTrapType = TRPM_SOFTWARE_INT;
7050 break;
7051
7052 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7053 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7054 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7055 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7056 enmTrapType = TRPM_TRAP;
7057 break;
7058
7059 default:
7060 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7061 enmTrapType = TRPM_32BIT_HACK;
7062 break;
7063 }
7064
7065 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7066
7067 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7068 AssertRC(rc);
7069
7070 if (fErrorCodeValid)
7071 TRPMSetErrorCode(pVCpu, uErrorCode);
7072
7073 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7074 && uVector == X86_XCPT_PF)
7075 {
7076 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7077 }
7078 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7079 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7080 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7081 {
7082 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7083 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
7084 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7085 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7086 }
7087
7088 /* Clear any pending events from the VMCS. */
7089 VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7090 VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, 0);
7091
7092 /* We're now done converting the pending event. */
7093 pVCpu->hm.s.Event.fPending = false;
7094}
7095
7096
7097/**
7098 * Does the necessary state syncing before returning to ring-3 for any reason
7099 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7100 *
7101 * @returns VBox status code.
7102 * @param pVCpu The cross context virtual CPU structure.
7103 * @param fImportState Whether to import the guest state from the VMCS back
7104 * to the guest-CPU context.
7105 *
7106 * @remarks No-long-jmp zone!!!
7107 */
7108static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
7109{
7110 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7111 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7112
7113 RTCPUID idCpu = RTMpCpuId();
7114 Log4Func(("HostCpuId=%u\n", idCpu));
7115
7116 /*
7117 * !!! IMPORTANT !!!
7118 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7119 */
7120
7121 /* Save the guest state if necessary. */
7122 if (fImportState)
7123 {
7124 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
7125 AssertRCReturn(rc, rc);
7126 }
7127
7128 /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
7129 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7130 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
7131
7132 /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
7133#ifdef VBOX_STRICT
7134 if (CPUMIsHyperDebugStateActive(pVCpu))
7135 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
7136#endif
7137 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7138 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7139 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7140
7141#if HC_ARCH_BITS == 64
7142 /* Restore host-state bits that VT-x only restores partially. */
7143 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7144 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7145 {
7146 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7147 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7148 }
7149 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7150#endif
7151
7152 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7153 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7154 {
7155 /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
7156 if (!fImportState)
7157 {
7158 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS);
7159 AssertRCReturn(rc, rc);
7160 }
7161 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7162 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7163 }
7164 else
7165 pVCpu->hm.s.vmx.fLazyMsrs = 0;
7166
7167 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7168 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7169
7170 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7171 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
7172 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
7173 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
7174 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
7175 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7176 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7177 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7178 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7179
7180 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7181
7182 /** @todo This partially defeats the purpose of having preemption hooks.
7183 * The problem is, deregistering the hooks should be moved to a place that
7184 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7185 * context.
7186 */
7187 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7188 {
7189 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7190 AssertRCReturn(rc, rc);
7191
7192 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7193 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7194 }
7195 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7196 NOREF(idCpu);
7197
7198 return VINF_SUCCESS;
7199}
7200
7201
7202/**
7203 * Leaves the VT-x session.
7204 *
7205 * @returns VBox status code.
7206 * @param pVCpu The cross context virtual CPU structure.
7207 *
7208 * @remarks No-long-jmp zone!!!
7209 */
7210static int hmR0VmxLeaveSession(PVMCPU pVCpu)
7211{
7212 HM_DISABLE_PREEMPT(pVCpu);
7213 HMVMX_ASSERT_CPU_SAFE(pVCpu);
7214 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7215 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7216
7217 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7218 and done this from the VMXR0ThreadCtxCallback(). */
7219 if (!pVCpu->hm.s.fLeaveDone)
7220 {
7221 int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
7222 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7223 pVCpu->hm.s.fLeaveDone = true;
7224 }
7225 Assert(!pVCpu->cpum.GstCtx.fExtrn);
7226
7227 /*
7228 * !!! IMPORTANT !!!
7229 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7230 */
7231
7232 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7233 /** @todo Deregistering here means we need to VMCLEAR always
7234 * (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
7235 * for calling VMMR0ThreadCtxHookDisable here! */
7236 VMMR0ThreadCtxHookDisable(pVCpu);
7237
7238 /* Leave HM context. This takes care of local init (term). */
7239 int rc = HMR0LeaveCpu(pVCpu);
7240
7241 HM_RESTORE_PREEMPT();
7242 return rc;
7243}
7244
7245
7246/**
7247 * Does the necessary state syncing before doing a longjmp to ring-3.
7248 *
7249 * @returns VBox status code.
7250 * @param pVCpu The cross context virtual CPU structure.
7251 *
7252 * @remarks No-long-jmp zone!!!
7253 */
7254DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
7255{
7256 return hmR0VmxLeaveSession(pVCpu);
7257}
7258
7259
7260/**
7261 * Take necessary actions before going back to ring-3.
7262 *
7263 * An action requires us to go back to ring-3. This function does the necessary
7264 * steps before we can safely return to ring-3. This is not the same as longjmps
7265 * to ring-3, this is voluntary and prepares the guest so it may continue
7266 * executing outside HM (recompiler/IEM).
7267 *
7268 * @returns VBox status code.
7269 * @param pVCpu The cross context virtual CPU structure.
7270 * @param rcExit The reason for exiting to ring-3. Can be
7271 * VINF_VMM_UNKNOWN_RING3_CALL.
7272 */
7273static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
7274{
7275 Assert(pVCpu);
7276 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7277
7278 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7279 {
7280 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7281 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7282 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7283 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7284 }
7285
7286 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7287 VMMRZCallRing3Disable(pVCpu);
7288 Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
7289
7290 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7291 if (pVCpu->hm.s.Event.fPending)
7292 {
7293 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7294 Assert(!pVCpu->hm.s.Event.fPending);
7295 }
7296
7297 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7298 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7299
7300 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7301 and if we're injecting an event we should have a TRPM trap pending. */
7302 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7303#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
7304 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7305#endif
7306
7307 /* Save guest state and restore host state bits. */
7308 int rc = hmR0VmxLeaveSession(pVCpu);
7309 AssertRCReturn(rc, rc);
7310 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7311 /* Thread-context hooks are unregistered at this point!!! */
7312
7313 /* Sync recompiler state. */
7314 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7315 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7316 | CPUM_CHANGED_LDTR
7317 | CPUM_CHANGED_GDTR
7318 | CPUM_CHANGED_IDTR
7319 | CPUM_CHANGED_TR
7320 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7321 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
7322 && CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
7323 {
7324 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7325 }
7326
7327 Assert(!pVCpu->hm.s.fClearTrapFlag);
7328
7329 /* Update the exit-to-ring 3 reason. */
7330 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
7331
7332 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7333 if (rcExit != VINF_EM_RAW_INTERRUPT)
7334 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7335
7336 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7337
7338 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7339 VMMRZCallRing3RemoveNotification(pVCpu);
7340 VMMRZCallRing3Enable(pVCpu);
7341
7342 return rc;
7343}
7344
7345
7346/**
7347 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7348 * longjump to ring-3 and possibly get preempted.
7349 *
7350 * @returns VBox status code.
7351 * @param pVCpu The cross context virtual CPU structure.
7352 * @param enmOperation The operation causing the ring-3 longjump.
7353 * @param pvUser User argument, currently unused, NULL.
7354 */
7355static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7356{
7357 RT_NOREF(pvUser);
7358 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7359 {
7360 /*
7361 * !!! IMPORTANT !!!
7362 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7363 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7364 */
7365 VMMRZCallRing3RemoveNotification(pVCpu);
7366 VMMRZCallRing3Disable(pVCpu);
7367 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7368 RTThreadPreemptDisable(&PreemptState);
7369
7370 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
7371 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7372 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7373
7374#if HC_ARCH_BITS == 64
7375 /* Restore host-state bits that VT-x only restores partially. */
7376 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7377 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7378 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7379 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7380#endif
7381
7382 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7383 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7384 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7385
7386 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7387 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7388 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7389 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7390 {
7391 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7392 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7393 }
7394
7395 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7396 VMMR0ThreadCtxHookDisable(pVCpu);
7397 HMR0LeaveCpu(pVCpu);
7398 RTThreadPreemptRestore(&PreemptState);
7399 return VINF_SUCCESS;
7400 }
7401
7402 Assert(pVCpu);
7403 Assert(pvUser);
7404 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7405 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7406
7407 VMMRZCallRing3Disable(pVCpu);
7408 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7409
7410 Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7411
7412 int rc = hmR0VmxLongJmpToRing3(pVCpu);
7413 AssertRCReturn(rc, rc);
7414
7415 VMMRZCallRing3Enable(pVCpu);
7416 return VINF_SUCCESS;
7417}
7418
7419
7420/**
7421 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7422 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7423 *
7424 * @param pVCpu The cross context virtual CPU structure.
7425 */
7426DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7427{
7428 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7429 {
7430 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7431 {
7432 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_PROC_CTLS_INT_WINDOW_EXIT;
7433 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7434 AssertRC(rc);
7435 Log4Func(("Setup interrupt-window exiting\n"));
7436 }
7437 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7438}
7439
7440
7441/**
7442 * Clears the interrupt-window exiting control in the VMCS.
7443 *
7444 * @param pVCpu The cross context virtual CPU structure.
7445 */
7446DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7447{
7448 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT);
7449 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
7450 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7451 AssertRC(rc);
7452 Log4Func(("Cleared interrupt-window exiting\n"));
7453}
7454
7455
7456/**
7457 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7458 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7459 *
7460 * @param pVCpu The cross context virtual CPU structure.
7461 */
7462DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7463{
7464 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7465 {
7466 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7467 {
7468 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7469 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7470 AssertRC(rc);
7471 Log4Func(("Setup NMI-window exiting\n"));
7472 }
7473 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7474}
7475
7476
7477/**
7478 * Clears the NMI-window exiting control in the VMCS.
7479 *
7480 * @param pVCpu The cross context virtual CPU structure.
7481 */
7482DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7483{
7484 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT);
7485 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7486 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7487 AssertRC(rc);
7488 Log4Func(("Cleared NMI-window exiting\n"));
7489}
7490
7491
7492/**
7493 * Evaluates the event to be delivered to the guest and sets it as the pending
7494 * event.
7495 *
7496 * @returns The VT-x guest-interruptibility state.
7497 * @param pVCpu The cross context virtual CPU structure.
7498 */
7499static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu)
7500{
7501 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7502 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7503 uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu);
7504 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7505 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7506 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7507
7508 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7509 Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7510 Assert(!fBlockSti || pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7511 Assert(!TRPMHasTrap(pVCpu));
7512
7513 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7514 APICUpdatePendingInterrupts(pVCpu);
7515
7516 /*
7517 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7518 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7519 */
7520 /** @todo SMI. SMIs take priority over NMIs. */
7521 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7522 {
7523 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7524 if ( !pVCpu->hm.s.Event.fPending
7525 && !fBlockNmi
7526 && !fBlockSti
7527 && !fBlockMovSS)
7528 {
7529 Log4Func(("Pending NMI\n"));
7530 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INT_INFO_VALID;
7531 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7532
7533 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7534 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7535 }
7536 else
7537 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7538 }
7539 /*
7540 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7541 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7542 */
7543 else if ( VMCPU_FF_IS_ANY_SET(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7544 && !pVCpu->hm.s.fSingleInstruction)
7545 {
7546 Assert(!DBGFIsStepping(pVCpu));
7547 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7548 AssertRCReturn(rc, 0);
7549 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7550 if ( !pVCpu->hm.s.Event.fPending
7551 && !fBlockInt
7552 && !fBlockSti
7553 && !fBlockMovSS)
7554 {
7555 uint8_t u8Interrupt;
7556 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7557 if (RT_SUCCESS(rc))
7558 {
7559 Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7560 uint32_t u32IntInfo = u8Interrupt
7561 | VMX_EXIT_INT_INFO_VALID
7562 | (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7563
7564 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7565 }
7566 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7567 {
7568 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
7569 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7570 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7571
7572 /*
7573 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7574 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7575 * need to re-set this force-flag here.
7576 */
7577 }
7578 else
7579 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7580 }
7581 else
7582 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7583 }
7584
7585 return fIntrState;
7586}
7587
7588
7589/**
7590 * Sets a pending-debug exception to be delivered to the guest if the guest is
7591 * single-stepping in the VMCS.
7592 *
7593 * @param pVCpu The cross context virtual CPU structure.
7594 */
7595DECLINLINE(int) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7596{
7597 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7598 RT_NOREF(pVCpu);
7599 return VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
7600}
7601
7602
7603/**
7604 * Injects any pending events into the guest if the guest is in a state to
7605 * receive them.
7606 *
7607 * @returns Strict VBox status code (i.e. informational status codes too).
7608 * @param pVCpu The cross context virtual CPU structure.
7609 * @param fIntrState The VT-x guest-interruptibility state.
7610 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7611 * return VINF_EM_DBG_STEPPED if the event was
7612 * dispatched directly.
7613 */
7614static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, uint32_t fIntrState, bool fStepping)
7615{
7616 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7617 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7618
7619 bool fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7620 bool fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7621
7622 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7623 Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7624 Assert(!fBlockSti || pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7625 Assert(!TRPMHasTrap(pVCpu));
7626
7627 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7628 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7629 if (pVCpu->hm.s.Event.fPending)
7630 {
7631 /*
7632 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7633 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7634 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7635 *
7636 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7637 */
7638 uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7639#ifdef VBOX_STRICT
7640 if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
7641 {
7642 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7643 Assert(!fBlockInt);
7644 Assert(!fBlockSti);
7645 Assert(!fBlockMovSS);
7646 }
7647 else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
7648 {
7649 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7650 Assert(!fBlockSti);
7651 Assert(!fBlockMovSS);
7652 Assert(!fBlockNmi);
7653 }
7654#endif
7655 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7656 uIntType));
7657 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7658 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7659 &fIntrState);
7660 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7661
7662 /* Update the interruptibility-state as it could have been changed by
7663 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7664 fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7665 fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7666
7667 if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
7668 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7669 else
7670 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7671 }
7672
7673 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7674 if ( fBlockSti
7675 || fBlockMovSS)
7676 {
7677 if (!pVCpu->hm.s.fSingleInstruction)
7678 {
7679 /*
7680 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7681 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7682 * See Intel spec. 27.3.4 "Saving Non-Register State".
7683 */
7684 Assert(!DBGFIsStepping(pVCpu));
7685 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7686 AssertRCReturn(rc, rc);
7687 if (pCtx->eflags.Bits.u1TF)
7688 {
7689 int rc2 = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7690 AssertRCReturn(rc2, rc2);
7691 }
7692 }
7693 else if (pCtx->eflags.Bits.u1TF)
7694 {
7695 /*
7696 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7697 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7698 */
7699 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
7700 fIntrState = 0;
7701 }
7702 }
7703
7704 /*
7705 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7706 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7707 */
7708 int rc3 = hmR0VmxExportGuestIntrState(pVCpu, fIntrState);
7709 AssertRCReturn(rc3, rc3);
7710
7711 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7712 NOREF(fBlockMovSS); NOREF(fBlockSti);
7713 return rcStrict;
7714}
7715
7716
7717/**
7718 * Injects a double-fault (\#DF) exception into the VM.
7719 *
7720 * @returns Strict VBox status code (i.e. informational status codes too).
7721 * @param pVCpu The cross context virtual CPU structure.
7722 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7723 * and should return VINF_EM_DBG_STEPPED if the event
7724 * is injected directly (register modified by us, not
7725 * by hardware on VM-entry).
7726 * @param pfIntrState Pointer to the current guest interruptibility-state.
7727 * This interruptibility-state will be updated if
7728 * necessary. This cannot not be NULL.
7729 */
7730DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, bool fStepping, uint32_t *pfIntrState)
7731{
7732 uint32_t const u32IntInfo = X86_XCPT_DF | VMX_EXIT_INT_INFO_VALID
7733 | (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT)
7734 | VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7735 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */, fStepping,
7736 pfIntrState);
7737}
7738
7739
7740/**
7741 * Injects a general-protection (\#GP) fault into the VM.
7742 *
7743 * @returns Strict VBox status code (i.e. informational status codes too).
7744 * @param pVCpu The cross context virtual CPU structure.
7745 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7746 * mode, i.e. in real-mode it's not valid).
7747 * @param u32ErrorCode The error code associated with the \#GP.
7748 * @param fStepping Whether we're running in
7749 * hmR0VmxRunGuestCodeStep() and should return
7750 * VINF_EM_DBG_STEPPED if the event is injected
7751 * directly (register modified by us, not by
7752 * hardware on VM-entry).
7753 * @param pfIntrState Pointer to the current guest interruptibility-state.
7754 * This interruptibility-state will be updated if
7755 * necessary. This cannot not be NULL.
7756 */
7757DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, bool fErrorCodeValid, uint32_t u32ErrorCode, bool fStepping,
7758 uint32_t *pfIntrState)
7759{
7760 uint32_t const u32IntInfo = X86_XCPT_GP | VMX_EXIT_INT_INFO_VALID
7761 | (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT)
7762 | (fErrorCodeValid ? VMX_EXIT_INT_INFO_ERROR_CODE_VALID : 0);
7763 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */, fStepping,
7764 pfIntrState);
7765}
7766
7767
7768/**
7769 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7770 * stack.
7771 *
7772 * @returns Strict VBox status code (i.e. informational status codes too).
7773 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7774 * @param pVCpu The cross context virtual CPU structure.
7775 * @param uValue The value to push to the guest stack.
7776 */
7777static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
7778{
7779 /*
7780 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7781 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7782 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7783 */
7784 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7785 if (pCtx->sp == 1)
7786 return VINF_EM_RESET;
7787 pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7788 int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
7789 AssertRC(rc);
7790 return rc;
7791}
7792
7793
7794/**
7795 * Injects an event into the guest upon VM-entry by updating the relevant fields
7796 * in the VM-entry area in the VMCS.
7797 *
7798 * @returns Strict VBox status code (i.e. informational status codes too).
7799 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7800 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7801 *
7802 * @param pVCpu The cross context virtual CPU structure.
7803 * @param u64IntInfo The VM-entry interruption-information field.
7804 * @param cbInstr The VM-entry instruction length in bytes (for
7805 * software interrupts, exceptions and privileged
7806 * software exceptions).
7807 * @param u32ErrCode The VM-entry exception error code.
7808 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7809 * @param pfIntrState Pointer to the current guest interruptibility-state.
7810 * This interruptibility-state will be updated if
7811 * necessary. This cannot not be NULL.
7812 * @param fStepping Whether we're running in
7813 * hmR0VmxRunGuestCodeStep() and should return
7814 * VINF_EM_DBG_STEPPED if the event is injected
7815 * directly (register modified by us, not by
7816 * hardware on VM-entry).
7817 */
7818static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7819 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7820{
7821 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7822 AssertMsg(!RT_HI_U32(u64IntInfo), ("%#RX64\n", u64IntInfo));
7823 Assert(pfIntrState);
7824
7825 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7826 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7827 uint32_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
7828 uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
7829
7830#ifdef VBOX_STRICT
7831 /*
7832 * Validate the error-code-valid bit for hardware exceptions.
7833 * No error codes for exceptions in real-mode.
7834 *
7835 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7836 */
7837 if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
7838 && !CPUMIsGuestInRealModeEx(pCtx))
7839 {
7840 switch (uVector)
7841 {
7842 case X86_XCPT_PF:
7843 case X86_XCPT_DF:
7844 case X86_XCPT_TS:
7845 case X86_XCPT_NP:
7846 case X86_XCPT_SS:
7847 case X86_XCPT_GP:
7848 case X86_XCPT_AC:
7849 AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
7850 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7851 RT_FALL_THRU();
7852 default:
7853 break;
7854 }
7855 }
7856#endif
7857
7858 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7859 Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
7860 || !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
7861
7862 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7863
7864 /*
7865 * Hardware interrupts & exceptions cannot be delivered through the software interrupt
7866 * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7867 * interrupt handler in the (real-mode) guest.
7868 *
7869 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7870 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7871 */
7872 if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
7873 {
7874 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7875 {
7876 /*
7877 * For unrestricted execution enabled CPUs running real-mode guests, we must not
7878 * set the deliver-error-code bit.
7879 *
7880 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7881 */
7882 u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
7883 }
7884 else
7885 {
7886 PVM pVM = pVCpu->CTX_SUFF(pVM);
7887 Assert(PDMVmmDevHeapIsEnabled(pVM));
7888 Assert(pVM->hm.s.vmx.pRealModeTSS);
7889
7890 /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7891 int rc2 = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_TABLE_MASK | CPUMCTX_EXTRN_RIP
7892 | CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_RFLAGS);
7893 AssertRCReturn(rc2, rc2);
7894
7895 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7896 size_t const cbIdtEntry = sizeof(X86IDTR16);
7897 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
7898 {
7899 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7900 if (uVector == X86_XCPT_DF)
7901 return VINF_EM_RESET;
7902
7903 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7904 if (uVector == X86_XCPT_GP)
7905 return hmR0VmxInjectXcptDF(pVCpu, fStepping, pfIntrState);
7906
7907 /*
7908 * If we're injecting an event with no valid IDT entry, inject a #GP.
7909 * No error codes for exceptions in real-mode.
7910 *
7911 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7912 */
7913 return hmR0VmxInjectXcptGP(pVCpu, false /* fErrCodeValid */, 0 /* u32ErrCode */, fStepping, pfIntrState);
7914 }
7915
7916 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7917 uint16_t uGuestIp = pCtx->ip;
7918 if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
7919 {
7920 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7921 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7922 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7923 }
7924 else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
7925 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7926
7927 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7928 X86IDTR16 IdtEntry;
7929 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
7930 rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7931 AssertRCReturn(rc2, rc2);
7932
7933 /* Construct the stack frame for the interrupt/exception handler. */
7934 VBOXSTRICTRC rcStrict;
7935 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
7936 if (rcStrict == VINF_SUCCESS)
7937 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
7938 if (rcStrict == VINF_SUCCESS)
7939 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
7940
7941 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7942 if (rcStrict == VINF_SUCCESS)
7943 {
7944 pCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7945 pCtx->rip = IdtEntry.offSel;
7946 pCtx->cs.Sel = IdtEntry.uSel;
7947 pCtx->cs.ValidSel = IdtEntry.uSel;
7948 pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7949 if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
7950 && uVector == X86_XCPT_PF)
7951 pCtx->cr2 = GCPtrFaultAddress;
7952
7953 /* If any other guest-state bits are changed here, make sure to update
7954 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7955 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS | HM_CHANGED_GUEST_CR2
7956 | HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
7957 | HM_CHANGED_GUEST_RSP);
7958
7959 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7960 if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
7961 {
7962 Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
7963 && uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
7964 Log4Func(("Clearing inhibition due to STI\n"));
7965 *pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
7966 }
7967 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7968 u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
7969
7970 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7971 it, if we are returning to ring-3 before executing guest code. */
7972 pVCpu->hm.s.Event.fPending = false;
7973
7974 /* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
7975 if (fStepping)
7976 rcStrict = VINF_EM_DBG_STEPPED;
7977 }
7978 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7979 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7980 return rcStrict;
7981 }
7982 }
7983
7984 /* Validate. */
7985 Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7986 Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
7987
7988 /* Inject. */
7989 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7990 if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
7991 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7992 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7993 AssertRCReturn(rc, rc);
7994
7995 /* Update CR2. */
7996 if ( VMX_ENTRY_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
7997 && uVector == X86_XCPT_PF)
7998 pCtx->cr2 = GCPtrFaultAddress;
7999
8000 Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8001
8002 return VINF_SUCCESS;
8003}
8004
8005
8006/**
8007 * Clears the interrupt-window exiting control in the VMCS and if necessary
8008 * clears the current event in the VMCS as well.
8009 *
8010 * @returns VBox status code.
8011 * @param pVCpu The cross context virtual CPU structure.
8012 *
8013 * @remarks Use this function only to clear events that have not yet been
8014 * delivered to the guest but are injected in the VMCS!
8015 * @remarks No-long-jump zone!!!
8016 */
8017static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8018{
8019 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8020 {
8021 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8022 Log4Func(("Cleared interrupt window\n"));
8023 }
8024
8025 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8026 {
8027 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8028 Log4Func(("Cleared NMI window\n"));
8029 }
8030}
8031
8032
8033/**
8034 * Enters the VT-x session.
8035 *
8036 * @returns VBox status code.
8037 * @param pVCpu The cross context virtual CPU structure.
8038 * @param pHostCpu Pointer to the global CPU info struct.
8039 */
8040VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu, PHMGLOBALCPUINFO pHostCpu)
8041{
8042 AssertPtr(pVCpu);
8043 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8044 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8045 RT_NOREF(pHostCpu);
8046
8047 LogFlowFunc(("pVCpu=%p\n", pVCpu));
8048 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8049 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8050
8051#ifdef VBOX_STRICT
8052 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8053 RTCCUINTREG uHostCR4 = ASMGetCR4();
8054 if (!(uHostCR4 & X86_CR4_VMXE))
8055 {
8056 LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8057 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8058 }
8059#endif
8060
8061 /*
8062 * Load the VCPU's VMCS as the current (and active) one.
8063 */
8064 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8065 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8066 if (RT_FAILURE(rc))
8067 return rc;
8068
8069 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8070 pVCpu->hm.s.fLeaveDone = false;
8071 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8072
8073 return VINF_SUCCESS;
8074}
8075
8076
8077/**
8078 * The thread-context callback (only on platforms which support it).
8079 *
8080 * @param enmEvent The thread-context event.
8081 * @param pVCpu The cross context virtual CPU structure.
8082 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8083 * @thread EMT(pVCpu)
8084 */
8085VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8086{
8087 NOREF(fGlobalInit);
8088
8089 switch (enmEvent)
8090 {
8091 case RTTHREADCTXEVENT_OUT:
8092 {
8093 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8094 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8095 VMCPU_ASSERT_EMT(pVCpu);
8096
8097 /* No longjmps (logger flushes, locks) in this fragile context. */
8098 VMMRZCallRing3Disable(pVCpu);
8099 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8100
8101 /*
8102 * Restore host-state (FPU, debug etc.)
8103 */
8104 if (!pVCpu->hm.s.fLeaveDone)
8105 {
8106 /*
8107 * Do -not- import the guest-state here as we might already be in the middle of importing
8108 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
8109 */
8110 hmR0VmxLeave(pVCpu, false /* fImportState */);
8111 pVCpu->hm.s.fLeaveDone = true;
8112 }
8113
8114 /* Leave HM context, takes care of local init (term). */
8115 int rc = HMR0LeaveCpu(pVCpu);
8116 AssertRC(rc); NOREF(rc);
8117
8118 /* Restore longjmp state. */
8119 VMMRZCallRing3Enable(pVCpu);
8120 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8121 break;
8122 }
8123
8124 case RTTHREADCTXEVENT_IN:
8125 {
8126 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8127 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8128 VMCPU_ASSERT_EMT(pVCpu);
8129
8130 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8131 VMMRZCallRing3Disable(pVCpu);
8132 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8133
8134 /* Initialize the bare minimum state required for HM. This takes care of
8135 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8136 int rc = hmR0EnterCpu(pVCpu);
8137 AssertRC(rc);
8138 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8139 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8140
8141 /* Load the active VMCS as the current one. */
8142 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8143 {
8144 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8145 AssertRC(rc); NOREF(rc);
8146 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8147 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8148 }
8149 pVCpu->hm.s.fLeaveDone = false;
8150
8151 /* Restore longjmp state. */
8152 VMMRZCallRing3Enable(pVCpu);
8153 break;
8154 }
8155
8156 default:
8157 break;
8158 }
8159}
8160
8161
8162/**
8163 * Exports the host state into the VMCS host-state area.
8164 * Sets up the VM-exit MSR-load area.
8165 *
8166 * The CPU state will be loaded from these fields on every successful VM-exit.
8167 *
8168 * @returns VBox status code.
8169 * @param pVCpu The cross context virtual CPU structure.
8170 *
8171 * @remarks No-long-jump zone!!!
8172 */
8173static int hmR0VmxExportHostState(PVMCPU pVCpu)
8174{
8175 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8176
8177 int rc = VINF_SUCCESS;
8178 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8179 {
8180 rc = hmR0VmxExportHostControlRegs();
8181 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8182
8183 rc = hmR0VmxExportHostSegmentRegs(pVCpu);
8184 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8185
8186 rc = hmR0VmxExportHostMsrs(pVCpu);
8187 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8188
8189 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
8190 }
8191 return rc;
8192}
8193
8194
8195/**
8196 * Saves the host state in the VMCS host-state.
8197 *
8198 * @returns VBox status code.
8199 * @param pVCpu The cross context virtual CPU structure.
8200 *
8201 * @remarks No-long-jump zone!!!
8202 */
8203VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
8204{
8205 AssertPtr(pVCpu);
8206 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8207
8208 /*
8209 * Export the host state here while entering HM context.
8210 * When thread-context hooks are used, we might get preempted and have to re-save the host
8211 * state but most of the time we won't be, so do it here before we disable interrupts.
8212 */
8213 return hmR0VmxExportHostState(pVCpu);
8214}
8215
8216
8217/**
8218 * Exports the guest state into the VMCS guest-state area.
8219 *
8220 * The will typically be done before VM-entry when the guest-CPU state and the
8221 * VMCS state may potentially be out of sync.
8222 *
8223 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8224 * VM-entry controls.
8225 * Sets up the appropriate VMX non-root function to execute guest code based on
8226 * the guest CPU mode.
8227 *
8228 * @returns VBox strict status code.
8229 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8230 * without unrestricted guest access and the VMMDev is not presently
8231 * mapped (e.g. EFI32).
8232 *
8233 * @param pVCpu The cross context virtual CPU structure.
8234 *
8235 * @remarks No-long-jump zone!!!
8236 */
8237static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu)
8238{
8239 AssertPtr(pVCpu);
8240 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8241
8242 LogFlowFunc(("pVCpu=%p\n", pVCpu));
8243
8244 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
8245
8246 /* Determine real-on-v86 mode. */
8247 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8248 if ( !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
8249 && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
8250 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8251
8252 /*
8253 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8254 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8255 */
8256 int rc = hmR0VmxSelectVMRunHandler(pVCpu);
8257 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8258
8259 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8260 rc = hmR0VmxExportGuestEntryCtls(pVCpu);
8261 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8262
8263 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8264 rc = hmR0VmxExportGuestExitCtls(pVCpu);
8265 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8266
8267 rc = hmR0VmxExportGuestCR0(pVCpu);
8268 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8269
8270 VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu);
8271 if (rcStrict == VINF_SUCCESS)
8272 { /* likely */ }
8273 else
8274 {
8275 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8276 return rcStrict;
8277 }
8278
8279 rc = hmR0VmxExportGuestSegmentRegs(pVCpu);
8280 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8281
8282 /* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8283 may alter controls if we determine we don't have to swap EFER after all. */
8284 rc = hmR0VmxExportGuestMsrs(pVCpu);
8285 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8286
8287 rc = hmR0VmxExportGuestApicTpr(pVCpu);
8288 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8289
8290 rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8291 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8292
8293 /* Exporting RFLAGS here is fine, even though RFLAGS.TF might depend on guest debug state which is
8294 not exported here. It is re-evaluated and updated if necessary in hmR0VmxExportSharedState(). */
8295 rc = hmR0VmxExportGuestRip(pVCpu);
8296 rc |= hmR0VmxExportGuestRsp(pVCpu);
8297 rc |= hmR0VmxExportGuestRflags(pVCpu);
8298 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8299
8300 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8301 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8302 | HM_CHANGED_GUEST_CR2
8303 | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8304 | HM_CHANGED_GUEST_X87
8305 | HM_CHANGED_GUEST_SSE_AVX
8306 | HM_CHANGED_GUEST_OTHER_XSAVE
8307 | HM_CHANGED_GUEST_XCRx
8308 | HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8309 | HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8310 | HM_CHANGED_GUEST_TSC_AUX
8311 | HM_CHANGED_GUEST_OTHER_MSRS
8312 | HM_CHANGED_GUEST_HWVIRT
8313 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8314
8315 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8316 return rc;
8317}
8318
8319
8320/**
8321 * Exports the state shared between the host and guest into the VMCS.
8322 *
8323 * @param pVCpu The cross context virtual CPU structure.
8324 *
8325 * @remarks No-long-jump zone!!!
8326 */
8327static void hmR0VmxExportSharedState(PVMCPU pVCpu)
8328{
8329 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8330 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8331
8332 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8333 {
8334 int rc = hmR0VmxExportSharedDebugState(pVCpu);
8335 AssertRC(rc);
8336 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8337
8338 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8339 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8340 {
8341 rc = hmR0VmxExportGuestRflags(pVCpu);
8342 AssertRC(rc);
8343 }
8344 }
8345
8346 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8347 {
8348 hmR0VmxLazyLoadGuestMsrs(pVCpu);
8349 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8350 }
8351
8352 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8353 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8354}
8355
8356
8357/**
8358 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8359 *
8360 * @returns Strict VBox status code (i.e. informational status codes too).
8361 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8362 * without unrestricted guest access and the VMMDev is not presently
8363 * mapped (e.g. EFI32).
8364 *
8365 * @param pVCpu The cross context virtual CPU structure.
8366 *
8367 * @remarks No-long-jump zone!!!
8368 */
8369static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu)
8370{
8371 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8372 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8373 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8374
8375#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8376 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8377#endif
8378
8379 /*
8380 * For many exits it's only RIP that changes and hence try to export it first
8381 * without going through a lot of change flag checks.
8382 */
8383 VBOXSTRICTRC rcStrict;
8384 uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8385 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8386 if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8387 {
8388 rcStrict = hmR0VmxExportGuestRip(pVCpu);
8389 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8390 { /* likely */}
8391 else
8392 AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8393 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8394 }
8395 else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8396 {
8397 rcStrict = hmR0VmxExportGuestState(pVCpu);
8398 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8399 { /* likely */}
8400 else
8401 {
8402 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8403 VBOXSTRICTRC_VAL(rcStrict)));
8404 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8405 return rcStrict;
8406 }
8407 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8408 }
8409 else
8410 rcStrict = VINF_SUCCESS;
8411
8412#ifdef VBOX_STRICT
8413 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8414 fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8415 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8416 AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8417 ("fCtxChanged=%#RX64\n", fCtxChanged));
8418#endif
8419 return rcStrict;
8420}
8421
8422
8423/**
8424 * Does the preparations before executing guest code in VT-x.
8425 *
8426 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8427 * recompiler/IEM. We must be cautious what we do here regarding committing
8428 * guest-state information into the VMCS assuming we assuredly execute the
8429 * guest in VT-x mode.
8430 *
8431 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8432 * the common-state (TRPM/forceflags), we must undo those changes so that the
8433 * recompiler/IEM can (and should) use them when it resumes guest execution.
8434 * Otherwise such operations must be done when we can no longer exit to ring-3.
8435 *
8436 * @returns Strict VBox status code (i.e. informational status codes too).
8437 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8438 * have been disabled.
8439 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8440 * double-fault into the guest.
8441 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8442 * dispatched directly.
8443 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8444 *
8445 * @param pVCpu The cross context virtual CPU structure.
8446 * @param pVmxTransient Pointer to the VMX transient structure.
8447 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8448 * us ignore some of the reasons for returning to
8449 * ring-3, and return VINF_EM_DBG_STEPPED if event
8450 * dispatching took place.
8451 */
8452static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
8453{
8454 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8455
8456#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_ONLY_IN_IEM
8457 Log2(("hmR0SvmPreRunGuest: Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
8458 return VINF_EM_RESCHEDULE_REM;
8459#endif
8460
8461#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8462 PGMRZDynMapFlushAutoSet(pVCpu);
8463#endif
8464
8465 /* Check force flag actions that might require us to go back to ring-3. */
8466 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
8467 if (rcStrict == VINF_SUCCESS)
8468 { /* FFs doesn't get set all the time. */ }
8469 else
8470 return rcStrict;
8471
8472 /*
8473 * Setup the virtualized-APIC accesses.
8474 *
8475 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8476 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8477 *
8478 * This is the reason we do it here and not in hmR0VmxExportGuestState().
8479 */
8480 PVM pVM = pVCpu->CTX_SUFF(pVM);
8481 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8482 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
8483 && PDMHasApic(pVM))
8484 {
8485 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8486 Assert(u64MsrApicBase);
8487 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8488
8489 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8490
8491 /* Unalias any existing mapping. */
8492 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8493 AssertRCReturn(rc, rc);
8494
8495 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8496 Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8497 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8498 AssertRCReturn(rc, rc);
8499
8500 /* Update the per-VCPU cache of the APIC base MSR. */
8501 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8502 }
8503
8504 if (TRPMHasTrap(pVCpu))
8505 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8506 uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu);
8507
8508 /*
8509 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
8510 * needs to be done with longjmps or interrupts + preemption enabled. Event injection might
8511 * also result in triple-faulting the VM.
8512 */
8513 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, fIntrState, fStepping);
8514 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8515 { /* likely */ }
8516 else
8517 {
8518 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8519 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8520 return rcStrict;
8521 }
8522
8523 /*
8524 * A longjump might result in importing CR3 even for VM-exits that don't necessarily
8525 * import CR3 themselves. We will need to update them here, as even as late as the above
8526 * hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
8527 * the below force flags to be set.
8528 */
8529 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8530 {
8531 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
8532 int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8533 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
8534 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
8535 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8536 }
8537 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8538 {
8539 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8540 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8541 }
8542
8543 /*
8544 * No longjmps to ring-3 from this point on!!!
8545 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8546 * This also disables flushing of the R0-logger instance (if any).
8547 */
8548 VMMRZCallRing3Disable(pVCpu);
8549
8550 /*
8551 * Export the guest state bits.
8552 *
8553 * We cannot perform longjmps while loading the guest state because we do not preserve the
8554 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8555 * CPU migration.
8556 *
8557 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8558 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8559 * Hence, loading of the guest state needs to be done -after- injection of events.
8560 */
8561 rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu);
8562 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8563 { /* likely */ }
8564 else
8565 {
8566 VMMRZCallRing3Enable(pVCpu);
8567 return rcStrict;
8568 }
8569
8570 /*
8571 * We disable interrupts so that we don't miss any interrupts that would flag preemption
8572 * (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
8573 * preemption disabled for a while. Since this is purly to aid the
8574 * RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
8575 * disable interrupt on NT.
8576 *
8577 * We need to check for force-flags that could've possible been altered since we last
8578 * checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
8579 * see @bugref{6398}).
8580 *
8581 * We also check a couple of other force-flags as a last opportunity to get the EMT back
8582 * to ring-3 before executing guest code.
8583 */
8584 pVmxTransient->fEFlags = ASMIntDisableFlags();
8585
8586 if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8587 && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8588 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8589 && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8590 {
8591 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8592 {
8593 pVCpu->hm.s.Event.fPending = false;
8594
8595 /*
8596 * We've injected any pending events. This is really the point of no return (to ring-3).
8597 *
8598 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8599 * returns from this function, so don't enable them here.
8600 */
8601 return VINF_SUCCESS;
8602 }
8603
8604 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
8605 rcStrict = VINF_EM_RAW_INTERRUPT;
8606 }
8607 else
8608 {
8609 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8610 rcStrict = VINF_EM_RAW_TO_R3;
8611 }
8612
8613 ASMSetFlags(pVmxTransient->fEFlags);
8614 VMMRZCallRing3Enable(pVCpu);
8615
8616 return rcStrict;
8617}
8618
8619
8620/**
8621 * Prepares to run guest code in VT-x and we've committed to doing so. This
8622 * means there is no backing out to ring-3 or anywhere else at this
8623 * point.
8624 *
8625 * @param pVCpu The cross context virtual CPU structure.
8626 * @param pVmxTransient Pointer to the VMX transient structure.
8627 *
8628 * @remarks Called with preemption disabled.
8629 * @remarks No-long-jump zone!!!
8630 */
8631static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
8632{
8633 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8634 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8635 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8636
8637 /*
8638 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8639 */
8640 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8641 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8642
8643 PVM pVM = pVCpu->CTX_SUFF(pVM);
8644 if (!CPUMIsGuestFPUStateActive(pVCpu))
8645 {
8646 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8647 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8648 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
8649 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8650 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8651 }
8652
8653 /*
8654 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8655 */
8656 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8657 && pVCpu->hm.s.vmx.cMsrs > 0)
8658 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8659
8660 /*
8661 * Re-save the host state bits as we may've been preempted (only happens when
8662 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8663 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8664 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8665 * See @bugref{8432}.
8666 */
8667 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8668 {
8669 int rc = hmR0VmxExportHostState(pVCpu);
8670 AssertRC(rc);
8671 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8672 }
8673 Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8674
8675 /*
8676 * Export the state shared between host and guest (FPU, debug, lazy MSRs).
8677 */
8678 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8679 hmR0VmxExportSharedState(pVCpu);
8680 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8681
8682 /* Store status of the shared guest-host state at the time of VM-entry. */
8683#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8684 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
8685 {
8686 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8687 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8688 }
8689 else
8690#endif
8691 {
8692 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8693 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8694 }
8695
8696 /*
8697 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8698 */
8699 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
8700 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8701
8702 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8703 RTCPUID idCurrentCpu = pCpu->idCpu;
8704 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8705 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8706 {
8707 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8708 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8709 }
8710
8711 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8712 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8713 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8714 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8715
8716 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8717
8718 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8719 to start executing. */
8720
8721 /*
8722 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8723 */
8724 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
8725 {
8726 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
8727 {
8728 bool fMsrUpdated;
8729 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8730 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8731 &fMsrUpdated);
8732 AssertRC(rc2);
8733 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8734 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8735 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8736 }
8737 else
8738 {
8739 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8740 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8741 }
8742 }
8743
8744 if (pVM->cpum.ro.GuestFeatures.fIbrs)
8745 {
8746 bool fMsrUpdated;
8747 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8748 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8749 &fMsrUpdated);
8750 AssertRC(rc2);
8751 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8752 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8753 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8754 }
8755
8756#ifdef VBOX_STRICT
8757 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8758 hmR0VmxCheckHostEferMsr(pVCpu);
8759 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8760#endif
8761#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8762 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
8763 {
8764 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
8765 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8766 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8767 }
8768#endif
8769}
8770
8771
8772/**
8773 * Performs some essential restoration of state after running guest code in
8774 * VT-x.
8775 *
8776 * @param pVCpu The cross context virtual CPU structure.
8777 * @param pVmxTransient Pointer to the VMX transient structure.
8778 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8779 *
8780 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8781 *
8782 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8783 * unconditionally when it is safe to do so.
8784 */
8785static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8786{
8787 uint64_t const uHostTsc = ASMReadTSC();
8788 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8789
8790 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8791 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8792 pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8793 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8794 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8795 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8796
8797 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
8798 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TscOffset);
8799
8800 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
8801 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8802 Assert(!ASMIntAreEnabled());
8803 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8804
8805#if HC_ARCH_BITS == 64
8806 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8807#endif
8808#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8809 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8810 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8811 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8812#else
8813 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8814#endif
8815#ifdef VBOX_STRICT
8816 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8817#endif
8818 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8819
8820 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8821 uint32_t uExitReason;
8822 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8823 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8824 AssertRC(rc);
8825 pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
8826 pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
8827
8828 if (rcVMRun == VINF_SUCCESS)
8829 {
8830 /*
8831 * Update the VM-exit history array here even if the VM-entry failed due to:
8832 * - Invalid guest state.
8833 * - MSR loading.
8834 * - Machine-check event.
8835 *
8836 * In any of the above cases we will still have a "valid" VM-exit reason
8837 * despite @a fVMEntryFailed being false.
8838 *
8839 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8840 *
8841 * Note! We don't have CS or RIP at this point. Will probably address that later
8842 * by amending the history entry added here.
8843 */
8844 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8845 UINT64_MAX, uHostTsc);
8846
8847 if (!pVmxTransient->fVMEntryFailed)
8848 {
8849 VMMRZCallRing3Enable(pVCpu);
8850
8851 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8852 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8853
8854#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8855 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8856 AssertRC(rc);
8857#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8858 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8859 AssertRC(rc);
8860#else
8861 /*
8862 * Import the guest-interruptibility state always as we need it while evaluating
8863 * injecting events on re-entry.
8864 *
8865 * We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8866 * checking for real-mode while exporting the state because all bits that cause
8867 * mode changes wrt CR0 are intercepted.
8868 */
8869 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8870 AssertRC(rc);
8871#endif
8872
8873 /*
8874 * Sync the TPR shadow with our APIC state.
8875 */
8876 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
8877 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8878 {
8879 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8880 AssertRC(rc);
8881 ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8882 }
8883
8884 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8885 return;
8886 }
8887 }
8888 else
8889 Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8890
8891 VMMRZCallRing3Enable(pVCpu);
8892}
8893
8894
8895/**
8896 * Runs the guest code using VT-x the normal way.
8897 *
8898 * @returns VBox status code.
8899 * @param pVCpu The cross context virtual CPU structure.
8900 *
8901 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8902 */
8903static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu)
8904{
8905 VMXTRANSIENT VmxTransient;
8906 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8907 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8908 uint32_t cLoops = 0;
8909
8910 for (;; cLoops++)
8911 {
8912 Assert(!HMR0SuspendPending());
8913 HMVMX_ASSERT_CPU_SAFE(pVCpu);
8914
8915 /* Preparatory work for running guest code, this may force us to return
8916 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8917 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8918 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
8919 if (rcStrict != VINF_SUCCESS)
8920 break;
8921
8922 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
8923 int rcRun = hmR0VmxRunGuest(pVCpu);
8924
8925 /* Restore any residual host-state and save any bits shared between host
8926 and guest into the guest-CPU state. Re-enables interrupts! */
8927 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8928
8929 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8930 if (RT_SUCCESS(rcRun))
8931 { /* very likely */ }
8932 else
8933 {
8934 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
8935 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
8936 return rcRun;
8937 }
8938
8939 /* Profile the VM-exit. */
8940 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8941 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8942 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8943 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
8944 HMVMX_START_EXIT_DISPATCH_PROF();
8945
8946 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
8947
8948 /* Handle the VM-exit. */
8949#ifdef HMVMX_USE_FUNCTION_TABLE
8950 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
8951#else
8952 rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient, VmxTransient.uExitReason);
8953#endif
8954 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
8955 if (rcStrict == VINF_SUCCESS)
8956 {
8957 if (cLoops <= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
8958 continue; /* likely */
8959 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8960 rcStrict = VINF_EM_RAW_INTERRUPT;
8961 }
8962 break;
8963 }
8964
8965 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8966 return rcStrict;
8967}
8968
8969
8970
8971/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8972 * probes.
8973 *
8974 * The following few functions and associated structure contains the bloat
8975 * necessary for providing detailed debug events and dtrace probes as well as
8976 * reliable host side single stepping. This works on the principle of
8977 * "subclassing" the normal execution loop and workers. We replace the loop
8978 * method completely and override selected helpers to add necessary adjustments
8979 * to their core operation.
8980 *
8981 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8982 * any performance for debug and analysis features.
8983 *
8984 * @{
8985 */
8986
8987/**
8988 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
8989 * the debug run loop.
8990 */
8991typedef struct VMXRUNDBGSTATE
8992{
8993 /** The RIP we started executing at. This is for detecting that we stepped. */
8994 uint64_t uRipStart;
8995 /** The CS we started executing with. */
8996 uint16_t uCsStart;
8997
8998 /** Whether we've actually modified the 1st execution control field. */
8999 bool fModifiedProcCtls : 1;
9000 /** Whether we've actually modified the 2nd execution control field. */
9001 bool fModifiedProcCtls2 : 1;
9002 /** Whether we've actually modified the exception bitmap. */
9003 bool fModifiedXcptBitmap : 1;
9004
9005 /** We desire the modified the CR0 mask to be cleared. */
9006 bool fClearCr0Mask : 1;
9007 /** We desire the modified the CR4 mask to be cleared. */
9008 bool fClearCr4Mask : 1;
9009 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9010 uint32_t fCpe1Extra;
9011 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9012 uint32_t fCpe1Unwanted;
9013 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9014 uint32_t fCpe2Extra;
9015 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9016 uint32_t bmXcptExtra;
9017 /** The sequence number of the Dtrace provider settings the state was
9018 * configured against. */
9019 uint32_t uDtraceSettingsSeqNo;
9020 /** VM-exits to check (one bit per VM-exit). */
9021 uint32_t bmExitsToCheck[3];
9022
9023 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9024 uint32_t fProcCtlsInitial;
9025 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9026 uint32_t fProcCtls2Initial;
9027 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9028 uint32_t bmXcptInitial;
9029} VMXRUNDBGSTATE;
9030AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9031typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9032
9033
9034/**
9035 * Initializes the VMXRUNDBGSTATE structure.
9036 *
9037 * @param pVCpu The cross context virtual CPU structure of the
9038 * calling EMT.
9039 * @param pDbgState The structure to initialize.
9040 */
9041static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9042{
9043 pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
9044 pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
9045
9046 pDbgState->fModifiedProcCtls = false;
9047 pDbgState->fModifiedProcCtls2 = false;
9048 pDbgState->fModifiedXcptBitmap = false;
9049 pDbgState->fClearCr0Mask = false;
9050 pDbgState->fClearCr4Mask = false;
9051 pDbgState->fCpe1Extra = 0;
9052 pDbgState->fCpe1Unwanted = 0;
9053 pDbgState->fCpe2Extra = 0;
9054 pDbgState->bmXcptExtra = 0;
9055 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9056 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9057 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9058}
9059
9060
9061/**
9062 * Updates the VMSC fields with changes requested by @a pDbgState.
9063 *
9064 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9065 * immediately before executing guest code, i.e. when interrupts are disabled.
9066 * We don't check status codes here as we cannot easily assert or return in the
9067 * latter case.
9068 *
9069 * @param pVCpu The cross context virtual CPU structure.
9070 * @param pDbgState The debug state.
9071 */
9072static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9073{
9074 /*
9075 * Ensure desired flags in VMCS control fields are set.
9076 * (Ignoring write failure here, as we're committed and it's just debug extras.)
9077 *
9078 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9079 * there should be no stale data in pCtx at this point.
9080 */
9081 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9082 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9083 {
9084 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
9085 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9086 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9087 Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9088 pDbgState->fModifiedProcCtls = true;
9089 }
9090
9091 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9092 {
9093 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9094 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9095 Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9096 pDbgState->fModifiedProcCtls2 = true;
9097 }
9098
9099 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9100 {
9101 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9102 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9103 Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9104 pDbgState->fModifiedXcptBitmap = true;
9105 }
9106
9107 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32Cr0Mask != 0)
9108 {
9109 pVCpu->hm.s.vmx.u32Cr0Mask = 0;
9110 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9111 Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9112 }
9113
9114 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32Cr4Mask != 0)
9115 {
9116 pVCpu->hm.s.vmx.u32Cr4Mask = 0;
9117 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9118 Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9119 }
9120}
9121
9122
9123/**
9124 * Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
9125 * re-entry next time around.
9126 *
9127 * @returns Strict VBox status code (i.e. informational status codes too).
9128 * @param pVCpu The cross context virtual CPU structure.
9129 * @param pDbgState The debug state.
9130 * @param rcStrict The return code from executing the guest using single
9131 * stepping.
9132 */
9133static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9134{
9135 /*
9136 * Restore VM-exit control settings as we may not reenter this function the
9137 * next time around.
9138 */
9139 /* We reload the initial value, trigger what we can of recalculations the
9140 next time around. From the looks of things, that's all that's required atm. */
9141 if (pDbgState->fModifiedProcCtls)
9142 {
9143 if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9144 pDbgState->fProcCtlsInitial |= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9145 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9146 AssertRCReturn(rc2, rc2);
9147 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9148 }
9149
9150 /* We're currently the only ones messing with this one, so just restore the
9151 cached value and reload the field. */
9152 if ( pDbgState->fModifiedProcCtls2
9153 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9154 {
9155 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9156 AssertRCReturn(rc2, rc2);
9157 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9158 }
9159
9160 /* If we've modified the exception bitmap, we restore it and trigger
9161 reloading and partial recalculation the next time around. */
9162 if (pDbgState->fModifiedXcptBitmap)
9163 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9164
9165 return rcStrict;
9166}
9167
9168
9169/**
9170 * Configures VM-exit controls for current DBGF and DTrace settings.
9171 *
9172 * This updates @a pDbgState and the VMCS execution control fields to reflect
9173 * the necessary VM-exits demanded by DBGF and DTrace.
9174 *
9175 * @param pVCpu The cross context virtual CPU structure.
9176 * @param pDbgState The debug state.
9177 * @param pVmxTransient Pointer to the VMX transient structure. May update
9178 * fUpdateTscOffsettingAndPreemptTimer.
9179 */
9180static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9181{
9182 /*
9183 * Take down the dtrace serial number so we can spot changes.
9184 */
9185 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9186 ASMCompilerBarrier();
9187
9188 /*
9189 * We'll rebuild most of the middle block of data members (holding the
9190 * current settings) as we go along here, so start by clearing it all.
9191 */
9192 pDbgState->bmXcptExtra = 0;
9193 pDbgState->fCpe1Extra = 0;
9194 pDbgState->fCpe1Unwanted = 0;
9195 pDbgState->fCpe2Extra = 0;
9196 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9197 pDbgState->bmExitsToCheck[i] = 0;
9198
9199 /*
9200 * Software interrupts (INT XXh) - no idea how to trigger these...
9201 */
9202 PVM pVM = pVCpu->CTX_SUFF(pVM);
9203 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9204 || VBOXVMM_INT_SOFTWARE_ENABLED())
9205 {
9206 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9207 }
9208
9209 /*
9210 * INT3 breakpoints - triggered by #BP exceptions.
9211 */
9212 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9213 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9214
9215 /*
9216 * Exception bitmap and XCPT events+probes.
9217 */
9218 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9219 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9220 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9221
9222 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9223 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9224 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9225 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9226 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9227 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9228 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9229 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9230 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9231 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9232 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9233 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9234 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9235 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9236 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9237 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9238 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9239 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9240
9241 if (pDbgState->bmXcptExtra)
9242 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9243
9244 /*
9245 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9246 *
9247 * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
9248 * So, when adding/changing/removing please don't forget to update it.
9249 *
9250 * Some of the macros are picking up local variables to save horizontal space,
9251 * (being able to see it in a table is the lesser evil here).
9252 */
9253#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9254 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9255 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9256#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9257 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9258 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9259 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9260 } else do { } while (0)
9261#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9262 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9263 { \
9264 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9265 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9266 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9267 } else do { } while (0)
9268#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9269 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9270 { \
9271 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9272 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9273 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9274 } else do { } while (0)
9275#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9276 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9277 { \
9278 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9279 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9280 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9281 } else do { } while (0)
9282
9283 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9284 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9285 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9286 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9287 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9288
9289 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9290 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9291 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9292 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9293 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
9294 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9295 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9296 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9297 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
9298 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9299 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
9300 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9301 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
9302 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9303 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9304 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9305 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9306 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9307 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9308 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9309 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9310 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9311 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9312 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9313 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9314 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9315 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9316 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9317 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9318 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9319 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9320 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9321 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9322 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9323 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9324 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9325
9326 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9327 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9328 {
9329 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_APIC_TPR);
9330 AssertRC(rc);
9331
9332#if 0 /** @todo fix me */
9333 pDbgState->fClearCr0Mask = true;
9334 pDbgState->fClearCr4Mask = true;
9335#endif
9336 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9337 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_STORE_EXIT | VMX_PROC_CTLS_CR8_STORE_EXIT;
9338 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9339 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_LOAD_EXIT | VMX_PROC_CTLS_CR8_LOAD_EXIT;
9340 pDbgState->fCpe1Unwanted |= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
9341 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9342 require clearing here and in the loop if we start using it. */
9343 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9344 }
9345 else
9346 {
9347 if (pDbgState->fClearCr0Mask)
9348 {
9349 pDbgState->fClearCr0Mask = false;
9350 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9351 }
9352 if (pDbgState->fClearCr4Mask)
9353 {
9354 pDbgState->fClearCr4Mask = false;
9355 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9356 }
9357 }
9358 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9359 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9360
9361 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9362 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9363 {
9364 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9365 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9366 }
9367 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9368 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9369
9370 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
9371 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9372 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
9373 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9374 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
9375 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9376 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
9377 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9378#if 0 /** @todo too slow, fix handler. */
9379 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
9380#endif
9381 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9382
9383 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9384 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9385 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9386 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9387 {
9388 pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
9389 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
9390 }
9391 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9392 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9393 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9394 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9395
9396 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9397 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9398 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9399 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9400 {
9401 pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
9402 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
9403 }
9404 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
9405 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
9406 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
9407 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
9408
9409 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9410 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9411 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
9412 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9413 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9414 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9415 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
9416 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9417 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9418 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9419 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
9420 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9421 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
9422 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9423 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9424 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9425 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
9426 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9427 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9428 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9429 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9430 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9431
9432#undef IS_EITHER_ENABLED
9433#undef SET_ONLY_XBM_IF_EITHER_EN
9434#undef SET_CPE1_XBM_IF_EITHER_EN
9435#undef SET_CPEU_XBM_IF_EITHER_EN
9436#undef SET_CPE2_XBM_IF_EITHER_EN
9437
9438 /*
9439 * Sanitize the control stuff.
9440 */
9441 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
9442 if (pDbgState->fCpe2Extra)
9443 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
9444 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
9445 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0;
9446 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
9447 {
9448 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9449 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9450 }
9451
9452 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9453 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9454 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9455 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9456}
9457
9458
9459/**
9460 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9461 * appropriate.
9462 *
9463 * The caller has checked the VM-exit against the
9464 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9465 * already, so we don't have to do that either.
9466 *
9467 * @returns Strict VBox status code (i.e. informational status codes too).
9468 * @param pVCpu The cross context virtual CPU structure.
9469 * @param pVmxTransient Pointer to the VMX-transient structure.
9470 * @param uExitReason The VM-exit reason.
9471 *
9472 * @remarks The name of this function is displayed by dtrace, so keep it short
9473 * and to the point. No longer than 33 chars long, please.
9474 */
9475static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9476{
9477 /*
9478 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9479 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9480 *
9481 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9482 * does. Must add/change/remove both places. Same ordering, please.
9483 *
9484 * Added/removed events must also be reflected in the next section
9485 * where we dispatch dtrace events.
9486 */
9487 bool fDtrace1 = false;
9488 bool fDtrace2 = false;
9489 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9490 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9491 uint32_t uEventArg = 0;
9492#define SET_EXIT(a_EventSubName) \
9493 do { \
9494 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9495 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9496 } while (0)
9497#define SET_BOTH(a_EventSubName) \
9498 do { \
9499 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9500 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9501 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9502 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9503 } while (0)
9504 switch (uExitReason)
9505 {
9506 case VMX_EXIT_MTF:
9507 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9508
9509 case VMX_EXIT_XCPT_OR_NMI:
9510 {
9511 uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9512 switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
9513 {
9514 case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
9515 case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
9516 case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
9517 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9518 {
9519 if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
9520 {
9521 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9522 uEventArg = pVmxTransient->uExitIntErrorCode;
9523 }
9524 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9525 switch (enmEvent1)
9526 {
9527 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9528 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9529 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9530 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9531 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9532 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9533 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9534 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9535 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9536 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9537 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9538 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9539 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9540 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9541 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9542 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9543 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9544 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9545 default: break;
9546 }
9547 }
9548 else
9549 AssertFailed();
9550 break;
9551
9552 case VMX_EXIT_INT_INFO_TYPE_SW_INT:
9553 uEventArg = idxVector;
9554 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9555 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9556 break;
9557 }
9558 break;
9559 }
9560
9561 case VMX_EXIT_TRIPLE_FAULT:
9562 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9563 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9564 break;
9565 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9566 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9567 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9568 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9569 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9570
9571 /* Instruction specific VM-exits: */
9572 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9573 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9574 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9575 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9576 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9577 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9578 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9579 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9580 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9581 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9582 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9583 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9584 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9585 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9586 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9587 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9588 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9589 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9590 case VMX_EXIT_MOV_CRX:
9591 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9592 if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9593 SET_BOTH(CRX_READ);
9594 else
9595 SET_BOTH(CRX_WRITE);
9596 uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
9597 break;
9598 case VMX_EXIT_MOV_DRX:
9599 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9600 if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
9601 == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9602 SET_BOTH(DRX_READ);
9603 else
9604 SET_BOTH(DRX_WRITE);
9605 uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
9606 break;
9607 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9608 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9609 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9610 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9611 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9612 case VMX_EXIT_GDTR_IDTR_ACCESS:
9613 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9614 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
9615 {
9616 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9617 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9618 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9619 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9620 }
9621 break;
9622
9623 case VMX_EXIT_LDTR_TR_ACCESS:
9624 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9625 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
9626 {
9627 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9628 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9629 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9630 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9631 }
9632 break;
9633
9634 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9635 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9636 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9637 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9638 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9639 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9640 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9641 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9642 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9643 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9644 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9645
9646 /* Events that aren't relevant at this point. */
9647 case VMX_EXIT_EXT_INT:
9648 case VMX_EXIT_INT_WINDOW:
9649 case VMX_EXIT_NMI_WINDOW:
9650 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9651 case VMX_EXIT_PREEMPT_TIMER:
9652 case VMX_EXIT_IO_INSTR:
9653 break;
9654
9655 /* Errors and unexpected events. */
9656 case VMX_EXIT_INIT_SIGNAL:
9657 case VMX_EXIT_SIPI:
9658 case VMX_EXIT_IO_SMI:
9659 case VMX_EXIT_SMI:
9660 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9661 case VMX_EXIT_ERR_MSR_LOAD:
9662 case VMX_EXIT_ERR_MACHINE_CHECK:
9663 break;
9664
9665 default:
9666 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9667 break;
9668 }
9669#undef SET_BOTH
9670#undef SET_EXIT
9671
9672 /*
9673 * Dtrace tracepoints go first. We do them here at once so we don't
9674 * have to copy the guest state saving and stuff a few dozen times.
9675 * Down side is that we've got to repeat the switch, though this time
9676 * we use enmEvent since the probes are a subset of what DBGF does.
9677 */
9678 if (fDtrace1 || fDtrace2)
9679 {
9680 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9681 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9682 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9683 switch (enmEvent1)
9684 {
9685 /** @todo consider which extra parameters would be helpful for each probe. */
9686 case DBGFEVENT_END: break;
9687 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
9688 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
9689 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
9690 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
9691 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
9692 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
9693 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
9694 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
9695 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
9696 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
9697 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
9698 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
9699 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
9700 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
9701 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
9702 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
9703 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
9704 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
9705 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9706 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9707 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
9708 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
9709 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
9710 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
9711 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
9712 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
9713 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
9714 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9715 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9716 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9717 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9718 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9719 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
9720 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9721 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
9722 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
9723 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
9724 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
9725 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
9726 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
9727 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
9728 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
9729 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
9730 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
9731 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
9732 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
9733 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
9734 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
9735 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
9736 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
9737 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
9738 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
9739 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
9740 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
9741 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
9742 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
9743 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
9744 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
9745 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
9746 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
9747 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
9748 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
9749 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
9750 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
9751 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
9752 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
9753 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9754 }
9755 switch (enmEvent2)
9756 {
9757 /** @todo consider which extra parameters would be helpful for each probe. */
9758 case DBGFEVENT_END: break;
9759 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
9760 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9761 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
9762 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
9763 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
9764 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
9765 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
9766 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
9767 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
9768 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9769 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9770 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9771 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9772 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9773 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
9774 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9775 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
9776 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
9777 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
9778 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
9779 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
9780 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
9781 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
9782 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
9783 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
9784 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
9785 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
9786 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
9787 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
9788 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
9789 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
9790 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
9791 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
9792 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
9793 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
9794 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
9795 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
9796 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
9797 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
9798 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
9799 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
9800 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
9801 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
9802 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
9803 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
9804 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
9805 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
9806 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
9807 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
9808 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
9809 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
9810 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
9811 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9812 }
9813 }
9814
9815 /*
9816 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9817 * the DBGF call will do a full check).
9818 *
9819 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9820 * Note! If we have to events, we prioritize the first, i.e. the instruction
9821 * one, in order to avoid event nesting.
9822 */
9823 PVM pVM = pVCpu->CTX_SUFF(pVM);
9824 if ( enmEvent1 != DBGFEVENT_END
9825 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9826 {
9827 HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
9828 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
9829 if (rcStrict != VINF_SUCCESS)
9830 return rcStrict;
9831 }
9832 else if ( enmEvent2 != DBGFEVENT_END
9833 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9834 {
9835 HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
9836 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
9837 if (rcStrict != VINF_SUCCESS)
9838 return rcStrict;
9839 }
9840
9841 return VINF_SUCCESS;
9842}
9843
9844
9845/**
9846 * Single-stepping VM-exit filtering.
9847 *
9848 * This is preprocessing the VM-exits and deciding whether we've gotten far
9849 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9850 * handling is performed.
9851 *
9852 * @returns Strict VBox status code (i.e. informational status codes too).
9853 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9854 * @param pVmxTransient Pointer to the VMX-transient structure.
9855 * @param pDbgState The debug state.
9856 */
9857DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
9858{
9859 /*
9860 * Expensive (saves context) generic dtrace VM-exit probe.
9861 */
9862 uint32_t const uExitReason = pVmxTransient->uExitReason;
9863 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9864 { /* more likely */ }
9865 else
9866 {
9867 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9868 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9869 AssertRC(rc);
9870 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
9871 }
9872
9873 /*
9874 * Check for host NMI, just to get that out of the way.
9875 */
9876 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9877 { /* normally likely */ }
9878 else
9879 {
9880 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9881 AssertRCReturn(rc2, rc2);
9882 uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
9883 if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
9884 return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
9885 }
9886
9887 /*
9888 * Check for single stepping event if we're stepping.
9889 */
9890 if (pVCpu->hm.s.fSingleInstruction)
9891 {
9892 switch (uExitReason)
9893 {
9894 case VMX_EXIT_MTF:
9895 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9896
9897 /* Various events: */
9898 case VMX_EXIT_XCPT_OR_NMI:
9899 case VMX_EXIT_EXT_INT:
9900 case VMX_EXIT_TRIPLE_FAULT:
9901 case VMX_EXIT_INT_WINDOW:
9902 case VMX_EXIT_NMI_WINDOW:
9903 case VMX_EXIT_TASK_SWITCH:
9904 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9905 case VMX_EXIT_APIC_ACCESS:
9906 case VMX_EXIT_EPT_VIOLATION:
9907 case VMX_EXIT_EPT_MISCONFIG:
9908 case VMX_EXIT_PREEMPT_TIMER:
9909
9910 /* Instruction specific VM-exits: */
9911 case VMX_EXIT_CPUID:
9912 case VMX_EXIT_GETSEC:
9913 case VMX_EXIT_HLT:
9914 case VMX_EXIT_INVD:
9915 case VMX_EXIT_INVLPG:
9916 case VMX_EXIT_RDPMC:
9917 case VMX_EXIT_RDTSC:
9918 case VMX_EXIT_RSM:
9919 case VMX_EXIT_VMCALL:
9920 case VMX_EXIT_VMCLEAR:
9921 case VMX_EXIT_VMLAUNCH:
9922 case VMX_EXIT_VMPTRLD:
9923 case VMX_EXIT_VMPTRST:
9924 case VMX_EXIT_VMREAD:
9925 case VMX_EXIT_VMRESUME:
9926 case VMX_EXIT_VMWRITE:
9927 case VMX_EXIT_VMXOFF:
9928 case VMX_EXIT_VMXON:
9929 case VMX_EXIT_MOV_CRX:
9930 case VMX_EXIT_MOV_DRX:
9931 case VMX_EXIT_IO_INSTR:
9932 case VMX_EXIT_RDMSR:
9933 case VMX_EXIT_WRMSR:
9934 case VMX_EXIT_MWAIT:
9935 case VMX_EXIT_MONITOR:
9936 case VMX_EXIT_PAUSE:
9937 case VMX_EXIT_GDTR_IDTR_ACCESS:
9938 case VMX_EXIT_LDTR_TR_ACCESS:
9939 case VMX_EXIT_INVEPT:
9940 case VMX_EXIT_RDTSCP:
9941 case VMX_EXIT_INVVPID:
9942 case VMX_EXIT_WBINVD:
9943 case VMX_EXIT_XSETBV:
9944 case VMX_EXIT_RDRAND:
9945 case VMX_EXIT_INVPCID:
9946 case VMX_EXIT_VMFUNC:
9947 case VMX_EXIT_RDSEED:
9948 case VMX_EXIT_XSAVES:
9949 case VMX_EXIT_XRSTORS:
9950 {
9951 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
9952 AssertRCReturn(rc, rc);
9953 if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
9954 || pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
9955 return VINF_EM_DBG_STEPPED;
9956 break;
9957 }
9958
9959 /* Errors and unexpected events: */
9960 case VMX_EXIT_INIT_SIGNAL:
9961 case VMX_EXIT_SIPI:
9962 case VMX_EXIT_IO_SMI:
9963 case VMX_EXIT_SMI:
9964 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9965 case VMX_EXIT_ERR_MSR_LOAD:
9966 case VMX_EXIT_ERR_MACHINE_CHECK:
9967 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9968 break;
9969
9970 default:
9971 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9972 break;
9973 }
9974 }
9975
9976 /*
9977 * Check for debugger event breakpoints and dtrace probes.
9978 */
9979 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9980 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9981 {
9982 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
9983 if (rcStrict != VINF_SUCCESS)
9984 return rcStrict;
9985 }
9986
9987 /*
9988 * Normal processing.
9989 */
9990#ifdef HMVMX_USE_FUNCTION_TABLE
9991 return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
9992#else
9993 return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
9994#endif
9995}
9996
9997
9998/**
9999 * Single steps guest code using VT-x.
10000 *
10001 * @returns Strict VBox status code (i.e. informational status codes too).
10002 * @param pVCpu The cross context virtual CPU structure.
10003 *
10004 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10005 */
10006static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu)
10007{
10008 VMXTRANSIENT VmxTransient;
10009 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10010
10011 /* Set HMCPU indicators. */
10012 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10013 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
10014 pVCpu->hm.s.fDebugWantRdTscExit = false;
10015 pVCpu->hm.s.fUsingDebugLoop = true;
10016
10017 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10018 VMXRUNDBGSTATE DbgState;
10019 hmR0VmxRunDebugStateInit(pVCpu, &DbgState);
10020 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
10021
10022 /*
10023 * The loop.
10024 */
10025 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10026 for (uint32_t cLoops = 0; ; cLoops++)
10027 {
10028 Assert(!HMR0SuspendPending());
10029 HMVMX_ASSERT_CPU_SAFE(pVCpu);
10030 bool fStepping = pVCpu->hm.s.fSingleInstruction;
10031
10032 /*
10033 * Preparatory work for running guest code, this may force us to return
10034 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10035 */
10036 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10037 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10038 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
10039 if (rcStrict != VINF_SUCCESS)
10040 break;
10041
10042 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10043 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10044
10045 /*
10046 * Now we can run the guest code.
10047 */
10048 int rcRun = hmR0VmxRunGuest(pVCpu);
10049
10050 /*
10051 * Restore any residual host-state and save any bits shared between host
10052 * and guest into the guest-CPU state. Re-enables interrupts!
10053 */
10054 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10055
10056 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10057 if (RT_SUCCESS(rcRun))
10058 { /* very likely */ }
10059 else
10060 {
10061 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10062 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10063 return rcRun;
10064 }
10065
10066 /* Profile the VM-exit. */
10067 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10068 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10069 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10070 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10071 HMVMX_START_EXIT_DISPATCH_PROF();
10072
10073 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10074
10075 /*
10076 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10077 */
10078 rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
10079 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10080 if (rcStrict != VINF_SUCCESS)
10081 break;
10082 if (cLoops > pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
10083 {
10084 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10085 rcStrict = VINF_EM_RAW_INTERRUPT;
10086 break;
10087 }
10088
10089 /*
10090 * Stepping: Did the RIP change, if so, consider it a single step.
10091 * Otherwise, make sure one of the TFs gets set.
10092 */
10093 if (fStepping)
10094 {
10095 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
10096 AssertRC(rc);
10097 if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
10098 || pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
10099 {
10100 rcStrict = VINF_EM_DBG_STEPPED;
10101 break;
10102 }
10103 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
10104 }
10105
10106 /*
10107 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10108 */
10109 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10110 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
10111 }
10112
10113 /*
10114 * Clear the X86_EFL_TF if necessary.
10115 */
10116 if (pVCpu->hm.s.fClearTrapFlag)
10117 {
10118 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
10119 AssertRC(rc);
10120 pVCpu->hm.s.fClearTrapFlag = false;
10121 pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
10122 }
10123 /** @todo there seems to be issues with the resume flag when the monitor trap
10124 * flag is pending without being used. Seen early in bios init when
10125 * accessing APIC page in protected mode. */
10126
10127 /*
10128 * Restore VM-exit control settings as we may not reenter this function the
10129 * next time around.
10130 */
10131 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10132
10133 /* Restore HMCPU indicators. */
10134 pVCpu->hm.s.fUsingDebugLoop = false;
10135 pVCpu->hm.s.fDebugWantRdTscExit = false;
10136 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10137
10138 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10139 return rcStrict;
10140}
10141
10142
10143/** @} */
10144
10145
10146/**
10147 * Checks if any expensive dtrace probes are enabled and we should go to the
10148 * debug loop.
10149 *
10150 * @returns true if we should use debug loop, false if not.
10151 */
10152static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10153{
10154 /* It's probably faster to OR the raw 32-bit counter variables together.
10155 Since the variables are in an array and the probes are next to one
10156 another (more or less), we have good locality. So, better read
10157 eight-nine cache lines ever time and only have one conditional, than
10158 128+ conditionals, right? */
10159 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10160 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10161 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10162 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10163 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10164 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10165 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10166 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10167 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10168 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10169 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10170 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10171 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10172 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10173 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10174 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10175 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10176 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10177 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10178 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10179 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10180 ) != 0
10181 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10182 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10183 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10184 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10185 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10186 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10187 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10188 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10189 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10190 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10191 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10192 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10193 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10194 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10195 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10196 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10197 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10198 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10199 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10200 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10201 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10202 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10203 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10204 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10205 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10206 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10207 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10208 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10209 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10210 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10211 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10212 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10213 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10214 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10215 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10216 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10217 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10218 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10219 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10220 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10221 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10222 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10223 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10224 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10225 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10226 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10227 ) != 0
10228 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10229 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10230 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10231 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10232 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10233 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10234 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10235 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10236 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10237 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10238 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10239 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10240 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10241 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10242 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10243 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10244 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10245 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10246 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10247 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10248 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10249 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10250 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10251 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10252 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10253 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10254 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10255 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10256 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10257 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10258 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10259 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10260 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10261 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10262 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10263 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10264 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10265 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10266 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10267 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10268 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10269 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10270 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10271 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10272 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10273 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10274 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10275 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10276 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10277 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10278 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10279 ) != 0;
10280}
10281
10282
10283/**
10284 * Runs the guest code using VT-x.
10285 *
10286 * @returns Strict VBox status code (i.e. informational status codes too).
10287 * @param pVCpu The cross context virtual CPU structure.
10288 */
10289VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
10290{
10291 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10292 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10293 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10294 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
10295
10296 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10297
10298 VBOXSTRICTRC rcStrict;
10299 if ( !pVCpu->hm.s.fUseDebugLoop
10300 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10301 && !DBGFIsStepping(pVCpu)
10302 && !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
10303 rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu);
10304 else
10305 rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu);
10306
10307 if (rcStrict == VERR_EM_INTERPRETER)
10308 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10309 else if (rcStrict == VINF_EM_RESET)
10310 rcStrict = VINF_EM_TRIPLE_FAULT;
10311
10312 int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
10313 if (RT_FAILURE(rc2))
10314 {
10315 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10316 rcStrict = rc2;
10317 }
10318 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10319 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10320 return rcStrict;
10321}
10322
10323
10324#ifndef HMVMX_USE_FUNCTION_TABLE
10325DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10326{
10327#ifdef DEBUG_ramshankar
10328#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10329 do { \
10330 if (a_fSave != 0) \
10331 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10332 VBOXSTRICTRC rcStrict = a_CallExpr; \
10333 if (a_fSave != 0) \
10334 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10335 return rcStrict; \
10336 } while (0)
10337#else
10338# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10339#endif
10340 switch (rcReason)
10341 {
10342 case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
10343 case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
10344 case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
10345 case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
10346 case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
10347 case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
10348 case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
10349 case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
10350 case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
10351 case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
10352 case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
10353 case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
10354 case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
10355 case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
10356 case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
10357 case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
10358 case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
10359 case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
10360 case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
10361 case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
10362 case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
10363 case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
10364 case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
10365 case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pVmxTransient));
10366 case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
10367 case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
10368 case VMX_EXIT_GDTR_IDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10369 case VMX_EXIT_LDTR_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10370 case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
10371 case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
10372 case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
10373 case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
10374 case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
10375 case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
10376
10377 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
10378 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
10379 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pVmxTransient);
10380 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pVmxTransient);
10381 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pVmxTransient);
10382 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pVmxTransient);
10383 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pVmxTransient);
10384 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
10385 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pVmxTransient);
10386
10387 case VMX_EXIT_VMCLEAR:
10388 case VMX_EXIT_VMLAUNCH:
10389 case VMX_EXIT_VMPTRLD:
10390 case VMX_EXIT_VMPTRST:
10391 case VMX_EXIT_VMREAD:
10392 case VMX_EXIT_VMRESUME:
10393 case VMX_EXIT_VMWRITE:
10394 case VMX_EXIT_VMXOFF:
10395 case VMX_EXIT_VMXON:
10396 case VMX_EXIT_INVEPT:
10397 case VMX_EXIT_INVVPID:
10398 case VMX_EXIT_VMFUNC:
10399 case VMX_EXIT_XSAVES:
10400 case VMX_EXIT_XRSTORS:
10401 return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
10402
10403 case VMX_EXIT_ENCLS:
10404 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10405 case VMX_EXIT_PML_FULL:
10406 default:
10407 return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
10408 }
10409#undef VMEXIT_CALL_RET
10410}
10411#endif /* !HMVMX_USE_FUNCTION_TABLE */
10412
10413
10414#ifdef VBOX_STRICT
10415/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10416# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10417 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10418
10419# define HMVMX_ASSERT_PREEMPT_CPUID() \
10420 do { \
10421 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10422 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10423 } while (0)
10424
10425# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10426 do { \
10427 AssertPtr((a_pVCpu)); \
10428 AssertPtr((a_pVmxTransient)); \
10429 Assert((a_pVmxTransient)->fVMEntryFailed == false); \
10430 Assert(ASMIntAreEnabled()); \
10431 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10432 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10433 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
10434 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10435 if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
10436 HMVMX_ASSERT_PREEMPT_CPUID(); \
10437 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10438 } while (0)
10439
10440# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10441 do { \
10442 Log4Func(("\n")); \
10443 } while (0)
10444#else
10445# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10446 do { \
10447 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10448 NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
10449 } while (0)
10450# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
10451#endif
10452
10453
10454/**
10455 * Advances the guest RIP by the specified number of bytes.
10456 *
10457 * @param pVCpu The cross context virtual CPU structure.
10458 * @param cbInstr Number of bytes to advance the RIP by.
10459 *
10460 * @remarks No-long-jump zone!!!
10461 */
10462DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
10463{
10464 /* Advance the RIP. */
10465 pVCpu->cpum.GstCtx.rip += cbInstr;
10466 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10467
10468 /* Update interrupt inhibition. */
10469 if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10470 && pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
10471 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10472}
10473
10474
10475/**
10476 * Advances the guest RIP after reading it from the VMCS.
10477 *
10478 * @returns VBox status code, no informational status codes.
10479 * @param pVCpu The cross context virtual CPU structure.
10480 * @param pVmxTransient Pointer to the VMX transient structure.
10481 *
10482 * @remarks No-long-jump zone!!!
10483 */
10484static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10485{
10486 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10487 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS);
10488 AssertRCReturn(rc, rc);
10489
10490 hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
10491
10492 /*
10493 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10494 * pending debug exception field as it takes care of priority of events.
10495 *
10496 * See Intel spec. 32.2.1 "Debug Exceptions".
10497 */
10498 if ( !pVCpu->hm.s.fSingleInstruction
10499 && pVCpu->cpum.GstCtx.eflags.Bits.u1TF)
10500 {
10501 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10502 AssertRCReturn(rc, rc);
10503 }
10504
10505 return VINF_SUCCESS;
10506}
10507
10508
10509/**
10510 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10511 * and update error record fields accordingly.
10512 *
10513 * @return VMX_IGS_* return codes.
10514 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10515 * wrong with the guest state.
10516 *
10517 * @param pVCpu The cross context virtual CPU structure.
10518 *
10519 * @remarks This function assumes our cache of the VMCS controls
10520 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10521 */
10522static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu)
10523{
10524#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10525#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10526 uError = (err); \
10527 break; \
10528 } else do { } while (0)
10529
10530 int rc;
10531 PVM pVM = pVCpu->CTX_SUFF(pVM);
10532 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10533 uint32_t uError = VMX_IGS_ERROR;
10534 uint32_t u32Val;
10535 bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10536
10537 do
10538 {
10539 /*
10540 * CR0.
10541 */
10542 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10543 uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10544 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10545 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10546 if (fUnrestrictedGuest)
10547 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
10548
10549 uint32_t u32GuestCr0;
10550 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10551 AssertRCBreak(rc);
10552 HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10553 HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10554 if ( !fUnrestrictedGuest
10555 && (u32GuestCr0 & X86_CR0_PG)
10556 && !(u32GuestCr0 & X86_CR0_PE))
10557 {
10558 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10559 }
10560
10561 /*
10562 * CR4.
10563 */
10564 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10565 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10566
10567 uint32_t u32GuestCr4;
10568 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10569 AssertRCBreak(rc);
10570 HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10571 HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10572
10573 /*
10574 * IA32_DEBUGCTL MSR.
10575 */
10576 uint64_t u64Val;
10577 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10578 AssertRCBreak(rc);
10579 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10580 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10581 {
10582 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10583 }
10584 uint64_t u64DebugCtlMsr = u64Val;
10585
10586#ifdef VBOX_STRICT
10587 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10588 AssertRCBreak(rc);
10589 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10590#endif
10591 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
10592
10593 /*
10594 * RIP and RFLAGS.
10595 */
10596 uint32_t u32Eflags;
10597#if HC_ARCH_BITS == 64
10598 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10599 AssertRCBreak(rc);
10600 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10601 if ( !fLongModeGuest
10602 || !pCtx->cs.Attr.n.u1Long)
10603 {
10604 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10605 }
10606 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10607 * must be identical if the "IA-32e mode guest" VM-entry
10608 * control is 1 and CS.L is 1. No check applies if the
10609 * CPU supports 64 linear-address bits. */
10610
10611 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10612 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10613 AssertRCBreak(rc);
10614 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10615 VMX_IGS_RFLAGS_RESERVED);
10616 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10617 u32Eflags = u64Val;
10618#else
10619 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10620 AssertRCBreak(rc);
10621 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10622 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10623#endif
10624
10625 if ( fLongModeGuest
10626 || ( fUnrestrictedGuest
10627 && !(u32GuestCr0 & X86_CR0_PE)))
10628 {
10629 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10630 }
10631
10632 uint32_t u32EntryInfo;
10633 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10634 AssertRCBreak(rc);
10635 if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
10636 && VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
10637 {
10638 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10639 }
10640
10641 /*
10642 * 64-bit checks.
10643 */
10644#if HC_ARCH_BITS == 64
10645 if (fLongModeGuest)
10646 {
10647 HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10648 HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10649 }
10650
10651 if ( !fLongModeGuest
10652 && (u32GuestCr4 & X86_CR4_PCIDE))
10653 {
10654 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10655 }
10656
10657 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10658 * 51:32 beyond the processor's physical-address width are 0. */
10659
10660 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10661 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10662 {
10663 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10664 }
10665
10666 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10667 AssertRCBreak(rc);
10668 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10669
10670 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10671 AssertRCBreak(rc);
10672 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10673#endif
10674
10675 /*
10676 * PERF_GLOBAL MSR.
10677 */
10678 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
10679 {
10680 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10681 AssertRCBreak(rc);
10682 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10683 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10684 }
10685
10686 /*
10687 * PAT MSR.
10688 */
10689 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
10690 {
10691 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10692 AssertRCBreak(rc);
10693 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10694 for (unsigned i = 0; i < 8; i++)
10695 {
10696 uint8_t u8Val = (u64Val & 0xff);
10697 if ( u8Val != 0 /* UC */
10698 && u8Val != 1 /* WC */
10699 && u8Val != 4 /* WT */
10700 && u8Val != 5 /* WP */
10701 && u8Val != 6 /* WB */
10702 && u8Val != 7 /* UC- */)
10703 {
10704 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10705 }
10706 u64Val >>= 8;
10707 }
10708 }
10709
10710 /*
10711 * EFER MSR.
10712 */
10713 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
10714 {
10715 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10716 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10717 AssertRCBreak(rc);
10718 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10719 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10720 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10721 & VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
10722 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10723 /** @todo r=ramshankar: Unrestricted check here is probably wrong, see
10724 * iemVmxVmentryCheckGuestState(). */
10725 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10726 || !(u32GuestCr0 & X86_CR0_PG)
10727 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10728 VMX_IGS_EFER_LMA_LME_MISMATCH);
10729 }
10730
10731 /*
10732 * Segment registers.
10733 */
10734 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10735 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10736 if (!(u32Eflags & X86_EFL_VM))
10737 {
10738 /* CS */
10739 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10740 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10741 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10742 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10743 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10744 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10745 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10746 /* CS cannot be loaded with NULL in protected mode. */
10747 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10748 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10749 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10750 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10751 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10752 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10753 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10754 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10755 else
10756 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10757
10758 /* SS */
10759 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10760 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10761 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10762 if ( !(pCtx->cr0 & X86_CR0_PE)
10763 || pCtx->cs.Attr.n.u4Type == 3)
10764 {
10765 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10766 }
10767 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10768 {
10769 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10770 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10771 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10772 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10773 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10774 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10775 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10776 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10777 }
10778
10779 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmenReg(). */
10780 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10781 {
10782 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10783 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10784 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10785 || pCtx->ds.Attr.n.u4Type > 11
10786 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10787 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10788 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10789 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10790 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10791 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10792 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10793 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10794 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10795 }
10796 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10797 {
10798 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10799 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10800 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10801 || pCtx->es.Attr.n.u4Type > 11
10802 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10803 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10804 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10805 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10806 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10807 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10808 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10809 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10810 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10811 }
10812 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10813 {
10814 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10815 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10816 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10817 || pCtx->fs.Attr.n.u4Type > 11
10818 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10819 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10820 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10821 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10822 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10823 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10824 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10825 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10826 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10827 }
10828 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10829 {
10830 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10831 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10832 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10833 || pCtx->gs.Attr.n.u4Type > 11
10834 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10835 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10836 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10837 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10838 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10839 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10840 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10841 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10842 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10843 }
10844 /* 64-bit capable CPUs. */
10845#if HC_ARCH_BITS == 64
10846 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10847 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10848 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10849 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10850 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10851 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10852 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10853 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10854 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10855 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10856 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10857#endif
10858 }
10859 else
10860 {
10861 /* V86 mode checks. */
10862 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10863 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10864 {
10865 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10866 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10867 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10868 }
10869 else
10870 {
10871 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10872 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10873 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10874 }
10875
10876 /* CS */
10877 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10878 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10879 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10880 /* SS */
10881 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10882 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10883 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10884 /* DS */
10885 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10886 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10887 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10888 /* ES */
10889 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10890 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10891 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10892 /* FS */
10893 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10894 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10895 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10896 /* GS */
10897 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10898 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10899 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10900 /* 64-bit capable CPUs. */
10901#if HC_ARCH_BITS == 64
10902 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10903 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10904 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10905 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10906 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10907 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10908 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10909 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10910 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10911 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10912 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10913#endif
10914 }
10915
10916 /*
10917 * TR.
10918 */
10919 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10920 /* 64-bit capable CPUs. */
10921#if HC_ARCH_BITS == 64
10922 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10923#endif
10924 if (fLongModeGuest)
10925 {
10926 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10927 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10928 }
10929 else
10930 {
10931 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10932 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10933 VMX_IGS_TR_ATTR_TYPE_INVALID);
10934 }
10935 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10936 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10937 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10938 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10939 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10940 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10941 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10942 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10943
10944 /*
10945 * GDTR and IDTR.
10946 */
10947#if HC_ARCH_BITS == 64
10948 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10949 AssertRCBreak(rc);
10950 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10951
10952 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10953 AssertRCBreak(rc);
10954 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10955#endif
10956
10957 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10958 AssertRCBreak(rc);
10959 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10960
10961 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10962 AssertRCBreak(rc);
10963 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10964
10965 /*
10966 * Guest Non-Register State.
10967 */
10968 /* Activity State. */
10969 uint32_t u32ActivityState;
10970 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10971 AssertRCBreak(rc);
10972 HMVMX_CHECK_BREAK( !u32ActivityState
10973 || (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
10974 VMX_IGS_ACTIVITY_STATE_INVALID);
10975 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10976 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10977 uint32_t u32IntrState;
10978 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
10979 AssertRCBreak(rc);
10980 if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
10981 || u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10982 {
10983 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10984 }
10985
10986 /** @todo Activity state and injecting interrupts. Left as a todo since we
10987 * currently don't use activity states but ACTIVE. */
10988
10989 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10990 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10991
10992 /* Guest interruptibility-state. */
10993 HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10994 HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
10995 != (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10996 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10997 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10998 || !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10999 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11000 if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
11001 {
11002 if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
11003 {
11004 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
11005 && !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
11006 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11007 }
11008 else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
11009 {
11010 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
11011 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11012 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
11013 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11014 }
11015 }
11016 /** @todo Assumes the processor is not in SMM. */
11017 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
11018 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11019 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
11020 || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
11021 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11022 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
11023 && VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
11024 && VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
11025 {
11026 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
11027 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11028 }
11029
11030 /* Pending debug exceptions. */
11031#if HC_ARCH_BITS == 64
11032 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
11033 AssertRCBreak(rc);
11034 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11035 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11036 u32Val = u64Val; /* For pending debug exceptions checks below. */
11037#else
11038 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
11039 AssertRCBreak(rc);
11040 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11041 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11042#endif
11043
11044 if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
11045 || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
11046 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11047 {
11048 if ( (u32Eflags & X86_EFL_TF)
11049 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11050 {
11051 /* Bit 14 is PendingDebug.BS. */
11052 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11053 }
11054 if ( !(u32Eflags & X86_EFL_TF)
11055 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11056 {
11057 /* Bit 14 is PendingDebug.BS. */
11058 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11059 }
11060 }
11061
11062 /* VMCS link pointer. */
11063 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11064 AssertRCBreak(rc);
11065 if (u64Val != UINT64_C(0xffffffffffffffff))
11066 {
11067 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11068 /** @todo Bits beyond the processor's physical-address width MBZ. */
11069 /** @todo 32-bit located in memory referenced by value of this field (as a
11070 * physical address) must contain the processor's VMCS revision ID. */
11071 /** @todo SMM checks. */
11072 }
11073
11074 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11075 * not using Nested Paging? */
11076 if ( pVM->hm.s.fNestedPaging
11077 && !fLongModeGuest
11078 && CPUMIsGuestInPAEModeEx(pCtx))
11079 {
11080 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11081 AssertRCBreak(rc);
11082 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11083
11084 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11085 AssertRCBreak(rc);
11086 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11087
11088 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11089 AssertRCBreak(rc);
11090 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11091
11092 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11093 AssertRCBreak(rc);
11094 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11095 }
11096
11097 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11098 if (uError == VMX_IGS_ERROR)
11099 uError = VMX_IGS_REASON_NOT_FOUND;
11100 } while (0);
11101
11102 pVCpu->hm.s.u32HMError = uError;
11103 return uError;
11104
11105#undef HMVMX_ERROR_BREAK
11106#undef HMVMX_CHECK_BREAK
11107}
11108
11109
11110/** @name VM-exit handlers.
11111 * @{
11112 */
11113/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11114/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11115/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11116
11117/**
11118 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11119 */
11120HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11121{
11122 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11123 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11124 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11125 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11126 return VINF_SUCCESS;
11127 return VINF_EM_RAW_INTERRUPT;
11128}
11129
11130
11131/**
11132 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11133 */
11134HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11135{
11136 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11137 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11138
11139 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11140 AssertRCReturn(rc, rc);
11141
11142 uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
11143 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
11144 && uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
11145 Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11146
11147 if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
11148 {
11149 /*
11150 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
11151 * injected it ourselves and anything we inject is not going to cause a VM-exit directly
11152 * for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
11153 *
11154 * [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11155 * [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
11156 */
11157 VMXDispatchHostNmi();
11158 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11159 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11160 return VINF_SUCCESS;
11161 }
11162
11163 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11164 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
11165 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11166 { /* likely */ }
11167 else
11168 {
11169 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11170 rcStrictRc1 = VINF_SUCCESS;
11171 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11172 return rcStrictRc1;
11173 }
11174
11175 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11176 uint32_t uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
11177 switch (uIntType)
11178 {
11179 case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11180 Assert(uVector == X86_XCPT_DB);
11181 RT_FALL_THRU();
11182 case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11183 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
11184 RT_FALL_THRU();
11185 case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11186 {
11187 /*
11188 * If there's any exception caused as a result of event injection, the resulting
11189 * secondary/final execption will be pending, we shall continue guest execution
11190 * after injecting the event. The page-fault case is complicated and we manually
11191 * handle any currently pending event in hmR0VmxExitXcptPF.
11192 */
11193 if (!pVCpu->hm.s.Event.fPending)
11194 { /* likely */ }
11195 else if (uVector != X86_XCPT_PF)
11196 {
11197 rc = VINF_SUCCESS;
11198 break;
11199 }
11200
11201 switch (uVector)
11202 {
11203 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
11204 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
11205 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
11206 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
11207 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
11208 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
11209
11210 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11211 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11212 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11213 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11214 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11215 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11216 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11217 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11218 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11219 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11220 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11221 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11222 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11223 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11224 default:
11225 {
11226 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11227 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11228 {
11229 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11230 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11231 Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
11232
11233 rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0);
11234 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11235 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11236 AssertRCReturn(rc, rc);
11237 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11238 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11239 0 /* GCPtrFaultAddress */);
11240 }
11241 else
11242 {
11243 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11244 pVCpu->hm.s.u32HMError = uVector;
11245 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11246 }
11247 break;
11248 }
11249 }
11250 break;
11251 }
11252
11253 default:
11254 {
11255 pVCpu->hm.s.u32HMError = uExitIntInfo;
11256 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11257 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
11258 break;
11259 }
11260 }
11261 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11262 return rc;
11263}
11264
11265
11266/**
11267 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11268 */
11269HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11270{
11271 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11272
11273 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11274 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11275
11276 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11277 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11278 return VINF_SUCCESS;
11279}
11280
11281
11282/**
11283 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11284 */
11285HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11286{
11287 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11288 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)))
11289 {
11290 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11291 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11292 }
11293
11294 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS));
11295
11296 /*
11297 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11298 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11299 */
11300 uint32_t fIntrState = 0;
11301 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
11302 AssertRCReturn(rc, rc);
11303
11304 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
11305 if ( fBlockSti
11306 && VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11307 {
11308 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11309 }
11310
11311 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11312 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11313
11314 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11315 return VINF_SUCCESS;
11316}
11317
11318
11319/**
11320 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11321 */
11322HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11323{
11324 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11325 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11326}
11327
11328
11329/**
11330 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11331 */
11332HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11333{
11334 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11335 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11336}
11337
11338
11339/**
11340 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11341 */
11342HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11343{
11344 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11345
11346 /*
11347 * Get the state we need and update the exit history entry.
11348 */
11349 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11350 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
11351 AssertRCReturn(rc, rc);
11352
11353 VBOXSTRICTRC rcStrict;
11354 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11355 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
11356 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11357 if (!pExitRec)
11358 {
11359 /*
11360 * Regular CPUID instruction execution.
11361 */
11362 rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
11363 if (rcStrict == VINF_SUCCESS)
11364 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11365 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11366 {
11367 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11368 rcStrict = VINF_SUCCESS;
11369 }
11370 }
11371 else
11372 {
11373 /*
11374 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
11375 */
11376 int rc2 = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11377 AssertRCReturn(rc2, rc2);
11378
11379 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11380 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11381
11382 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11383 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11384
11385 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11386 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11387 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11388 }
11389 return rcStrict;
11390}
11391
11392
11393/**
11394 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11395 */
11396HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11397{
11398 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11399 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR4);
11400 AssertRCReturn(rc, rc);
11401
11402 if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
11403 return VINF_EM_RAW_EMULATE_INSTR;
11404
11405 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11406 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11407}
11408
11409
11410/**
11411 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11412 */
11413HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11414{
11415 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11416 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11417 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11418 AssertRCReturn(rc, rc);
11419
11420 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11421 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11422 {
11423 /* If we get a spurious VM-exit when offsetting is enabled,
11424 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11425 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
11426 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11427 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11428 }
11429 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11430 {
11431 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11432 rcStrict = VINF_SUCCESS;
11433 }
11434 return rcStrict;
11435}
11436
11437
11438/**
11439 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11440 */
11441HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11442{
11443 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11444 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_TSC_AUX);
11445 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11446 AssertRCReturn(rc, rc);
11447
11448 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11449 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11450 {
11451 /* If we get a spurious VM-exit when offsetting is enabled,
11452 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11453 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
11454 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11455 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11456 }
11457 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11458 {
11459 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11460 rcStrict = VINF_SUCCESS;
11461 }
11462 return rcStrict;
11463}
11464
11465
11466/**
11467 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11468 */
11469HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11470{
11471 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11472 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11473 AssertRCReturn(rc, rc);
11474
11475 PVM pVM = pVCpu->CTX_SUFF(pVM);
11476 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11477 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11478 if (RT_LIKELY(rc == VINF_SUCCESS))
11479 {
11480 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11481 Assert(pVmxTransient->cbInstr == 2);
11482 }
11483 else
11484 {
11485 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11486 rc = VERR_EM_INTERPRETER;
11487 }
11488 return rc;
11489}
11490
11491
11492/**
11493 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11494 */
11495HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11496{
11497 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11498
11499 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11500 if (EMAreHypercallInstructionsEnabled(pVCpu))
11501 {
11502 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CR0
11503 | CPUMCTX_EXTRN_SS | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_EFER);
11504 AssertRCReturn(rc, rc);
11505
11506 /* Perform the hypercall. */
11507 rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
11508 if (rcStrict == VINF_SUCCESS)
11509 {
11510 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11511 AssertRCReturn(rc, rc);
11512 }
11513 else
11514 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11515 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11516 || RT_FAILURE(rcStrict));
11517
11518 /* If the hypercall changes anything other than guest's general-purpose registers,
11519 we would need to reload the guest changed bits here before VM-entry. */
11520 }
11521 else
11522 Log4Func(("Hypercalls not enabled\n"));
11523
11524 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11525 if (RT_FAILURE(rcStrict))
11526 {
11527 hmR0VmxSetPendingXcptUD(pVCpu);
11528 rcStrict = VINF_SUCCESS;
11529 }
11530
11531 return rcStrict;
11532}
11533
11534
11535/**
11536 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11537 */
11538HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11539{
11540 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11541 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11542
11543 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11544 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11545 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
11546 AssertRCReturn(rc, rc);
11547
11548 VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
11549
11550 if (rcStrict == VINF_SUCCESS || rcStrict == VINF_PGM_SYNC_CR3)
11551 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11552 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11553 {
11554 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11555 rcStrict = VINF_SUCCESS;
11556 }
11557 else
11558 AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) sttus: %Rrc\n", pVmxTransient->uExitQual,
11559 VBOXSTRICTRC_VAL(rcStrict)));
11560 return rcStrict;
11561}
11562
11563
11564/**
11565 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11566 */
11567HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11568{
11569 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11570 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11571 AssertRCReturn(rc, rc);
11572
11573 PVM pVM = pVCpu->CTX_SUFF(pVM);
11574 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11575 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11576 if (RT_LIKELY(rc == VINF_SUCCESS))
11577 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11578 else
11579 {
11580 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11581 rc = VERR_EM_INTERPRETER;
11582 }
11583 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11584 return rc;
11585}
11586
11587
11588/**
11589 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11590 */
11591HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11592{
11593 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11594 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11595 AssertRCReturn(rc, rc);
11596
11597 PVM pVM = pVCpu->CTX_SUFF(pVM);
11598 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11599 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11600 rc = VBOXSTRICTRC_VAL(rc2);
11601 if (RT_LIKELY( rc == VINF_SUCCESS
11602 || rc == VINF_EM_HALT))
11603 {
11604 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11605 AssertRCReturn(rc3, rc3);
11606
11607 if ( rc == VINF_EM_HALT
11608 && EMMonitorWaitShouldContinue(pVCpu, pCtx))
11609 rc = VINF_SUCCESS;
11610 }
11611 else
11612 {
11613 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11614 rc = VERR_EM_INTERPRETER;
11615 }
11616 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11617 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11618 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11619 return rc;
11620}
11621
11622
11623/**
11624 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11625 */
11626HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11627{
11628 /*
11629 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11630 * mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11631 * treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11632 * VMX root operation. If we get here, something funny is going on.
11633 *
11634 * See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11635 */
11636 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11637 AssertMsgFailed(("Unexpected RSM VM-exit\n"));
11638 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11639}
11640
11641
11642/**
11643 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11644 */
11645HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11646{
11647 /*
11648 * This can only happen if we support dual-monitor treatment of SMI, which can be activated
11649 * by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11650 * this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11651 * an SMI. If we get here, something funny is going on.
11652 *
11653 * See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11654 * See Intel spec. 25.3 "Other Causes of VM-Exits"
11655 */
11656 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11657 AssertMsgFailed(("Unexpected SMI VM-exit\n"));
11658 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11659}
11660
11661
11662/**
11663 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11664 */
11665HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11666{
11667 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11668 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11669 AssertMsgFailed(("Unexpected IO SMI VM-exit\n"));
11670 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11671}
11672
11673
11674/**
11675 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11676 */
11677HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11678{
11679 /*
11680 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11681 * We don't make use of it as our guests don't have direct access to the host LAPIC.
11682 * See Intel spec. 25.3 "Other Causes of VM-exits".
11683 */
11684 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11685 AssertMsgFailed(("Unexpected SIPI VM-exit\n"));
11686 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11687}
11688
11689
11690/**
11691 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11692 * VM-exit.
11693 */
11694HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11695{
11696 /*
11697 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11698 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11699 *
11700 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11701 * See Intel spec. "23.8 Restrictions on VMX operation".
11702 */
11703 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11704 return VINF_SUCCESS;
11705}
11706
11707
11708/**
11709 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11710 * VM-exit.
11711 */
11712HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11713{
11714 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11715 return VINF_EM_RESET;
11716}
11717
11718
11719/**
11720 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11721 */
11722HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11723{
11724 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11725 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_HLT_EXIT);
11726
11727 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11728 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RFLAGS);
11729 AssertRCReturn(rc, rc);
11730
11731 if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
11732 rc = VINF_SUCCESS;
11733 else
11734 rc = VINF_EM_HALT;
11735
11736 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11737 if (rc != VINF_SUCCESS)
11738 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11739 return rc;
11740}
11741
11742
11743/**
11744 * VM-exit handler for instructions that result in a \#UD exception delivered to
11745 * the guest.
11746 */
11747HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11748{
11749 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11750 hmR0VmxSetPendingXcptUD(pVCpu);
11751 return VINF_SUCCESS;
11752}
11753
11754
11755/**
11756 * VM-exit handler for expiry of the VMX preemption timer.
11757 */
11758HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11759{
11760 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11761
11762 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11763 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11764
11765 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11766 PVM pVM = pVCpu->CTX_SUFF(pVM);
11767 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11768 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11769 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11770}
11771
11772
11773/**
11774 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11775 */
11776HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11777{
11778 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11779
11780 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11781 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_CR4);
11782 AssertRCReturn(rc, rc);
11783
11784 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11785 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11786 : HM_CHANGED_RAISED_XCPT_MASK);
11787
11788 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11789 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
11790
11791 return rcStrict;
11792}
11793
11794
11795/**
11796 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11797 */
11798HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11799{
11800 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11801 /** @todo Use VM-exit instruction information. */
11802 return VERR_EM_INTERPRETER;
11803}
11804
11805
11806/**
11807 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11808 * Error VM-exit.
11809 */
11810HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11811{
11812 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11813 AssertRCReturn(rc, rc);
11814 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11815 if (RT_FAILURE(rc))
11816 return rc;
11817
11818 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
11819 NOREF(uInvalidReason);
11820
11821#ifdef VBOX_STRICT
11822 uint32_t fIntrState;
11823 RTHCUINTREG uHCReg;
11824 uint64_t u64Val;
11825 uint32_t u32Val;
11826
11827 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11828 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11829 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11830 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
11831 AssertRCReturn(rc, rc);
11832
11833 Log4(("uInvalidReason %u\n", uInvalidReason));
11834 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11835 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11836 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11837 Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
11838
11839 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11840 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11841 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11842 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11843 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11844 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11845 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11846 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11847 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11848 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11849 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11850 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11851
11852 hmR0DumpRegs(pVCpu);
11853#else
11854 NOREF(pVmxTransient);
11855#endif
11856
11857 return VERR_VMX_INVALID_GUEST_STATE;
11858}
11859
11860
11861/**
11862 * VM-exit handler for VM-entry failure due to an MSR-load
11863 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11864 */
11865HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11866{
11867 AssertMsgFailed(("Unexpected MSR-load exit\n"));
11868 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11869}
11870
11871
11872/**
11873 * VM-exit handler for VM-entry failure due to a machine-check event
11874 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11875 */
11876HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11877{
11878 AssertMsgFailed(("Unexpected machine-check event exit\n"));
11879 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11880}
11881
11882
11883/**
11884 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11885 * theory.
11886 */
11887HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11888{
11889 RT_NOREF2(pVCpu, pVmxTransient);
11890 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d\n", pVmxTransient->uExitReason));
11891 return VERR_VMX_UNDEFINED_EXIT_CODE;
11892}
11893
11894
11895/**
11896 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11897 * (VMX_EXIT_GDTR_IDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11898 * Conditional VM-exit.
11899 */
11900HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11901{
11902 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11903
11904 /* By default, we don't enable VMX_PROC_CTLS2_DESCRIPTOR_TABLE_EXIT. */
11905 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11906 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_DESC_TABLE_EXIT)
11907 return VERR_EM_INTERPRETER;
11908 AssertMsgFailed(("Unexpected XDTR access\n"));
11909 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11910}
11911
11912
11913/**
11914 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11915 */
11916HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11917{
11918 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11919
11920 /* By default, we don't enable VMX_PROC_CTLS2_RDRAND_EXIT. */
11921 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_RDRAND_EXIT)
11922 return VERR_EM_INTERPRETER;
11923 AssertMsgFailed(("Unexpected RDRAND exit\n"));
11924 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11925}
11926
11927
11928/**
11929 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11930 */
11931HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11932{
11933 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11934
11935 /** @todo Optimize this: We currently drag in in the whole MSR state
11936 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11937 * MSRs required. That would require changes to IEM and possibly CPUM too.
11938 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11939 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
11940 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11941 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
11942 switch (idMsr)
11943 {
11944 /* The FS and GS base MSRs are not part of the above all-MSRs mask. */
11945 case MSR_K8_FS_BASE: rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_FS); break;
11946 case MSR_K8_GS_BASE: rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_GS); break;
11947 }
11948 AssertRCReturn(rc, rc);
11949
11950 Log4Func(("ecx=%#RX32\n", idMsr));
11951
11952#ifdef VBOX_STRICT
11953 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
11954 {
11955 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr)
11956 && idMsr != MSR_K6_EFER)
11957 {
11958 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
11959 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11960 }
11961 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11962 {
11963 VMXMSREXITREAD enmRead;
11964 VMXMSREXITWRITE enmWrite;
11965 int rc2 = HMVmxGetMsrPermission(pVCpu->hm.s.vmx.pvMsrBitmap, idMsr, &enmRead, &enmWrite);
11966 AssertRCReturn(rc2, rc2);
11967 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11968 {
11969 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
11970 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11971 }
11972 }
11973 }
11974#endif
11975
11976 VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
11977 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11978 if (rcStrict == VINF_SUCCESS)
11979 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11980 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX);
11981 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11982 {
11983 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11984 rcStrict = VINF_SUCCESS;
11985 }
11986 else
11987 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11988
11989 return rcStrict;
11990}
11991
11992
11993/**
11994 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11995 */
11996HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11997{
11998 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11999
12000 /** @todo Optimize this: We currently drag in in the whole MSR state
12001 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
12002 * MSRs required. That would require changes to IEM and possibly CPUM too.
12003 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
12004 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
12005 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12006 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK
12007 | CPUMCTX_EXTRN_ALL_MSRS);
12008 switch (idMsr)
12009 {
12010 /*
12011 * The FS and GS base MSRs are not part of the above all-MSRs mask.
12012 *
12013 * Although we don't need to fetch the base as it will be overwritten shortly, while
12014 * loading guest-state we would also load the entire segment register including limit
12015 * and attributes and thus we need to load them here.
12016 */
12017 case MSR_K8_FS_BASE: rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_FS); break;
12018 case MSR_K8_GS_BASE: rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_GS); break;
12019 }
12020 AssertRCReturn(rc, rc);
12021
12022 Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
12023
12024 VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
12025 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12026
12027 if (rcStrict == VINF_SUCCESS)
12028 {
12029 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12030
12031 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
12032 if ( idMsr == MSR_IA32_APICBASE
12033 || ( idMsr >= MSR_IA32_X2APIC_START
12034 && idMsr <= MSR_IA32_X2APIC_END))
12035 {
12036 /*
12037 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12038 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before IEM changes it.
12039 */
12040 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
12041 }
12042 else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12043 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12044 else if (idMsr == MSR_K6_EFER)
12045 {
12046 /*
12047 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12048 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12049 * the other bits as well, SCE and NXE. See @bugref{7368}.
12050 */
12051 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS
12052 | HM_CHANGED_VMX_EXIT_CTLS);
12053 }
12054
12055 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12056 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
12057 {
12058 switch (idMsr)
12059 {
12060 case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
12061 case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
12062 case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
12063 case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
12064 case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
12065 case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
12066 default:
12067 {
12068 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
12069 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12070 else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
12071 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
12072 break;
12073 }
12074 }
12075 }
12076#ifdef VBOX_STRICT
12077 else
12078 {
12079 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12080 switch (idMsr)
12081 {
12082 case MSR_IA32_SYSENTER_CS:
12083 case MSR_IA32_SYSENTER_EIP:
12084 case MSR_IA32_SYSENTER_ESP:
12085 case MSR_K8_FS_BASE:
12086 case MSR_K8_GS_BASE:
12087 {
12088 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
12089 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12090 }
12091
12092 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12093 default:
12094 {
12095 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
12096 {
12097 /* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
12098 if (idMsr != MSR_K6_EFER)
12099 {
12100 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12101 idMsr));
12102 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12103 }
12104 }
12105
12106 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
12107 {
12108 VMXMSREXITREAD enmRead;
12109 VMXMSREXITWRITE enmWrite;
12110 int rc2 = HMVmxGetMsrPermission(pVCpu->hm.s.vmx.pvMsrBitmap, idMsr, &enmRead, &enmWrite);
12111 AssertRCReturn(rc2, rc2);
12112 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12113 {
12114 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
12115 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12116 }
12117 }
12118 break;
12119 }
12120 }
12121 }
12122#endif /* VBOX_STRICT */
12123 }
12124 else if (rcStrict == VINF_IEM_RAISED_XCPT)
12125 {
12126 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
12127 rcStrict = VINF_SUCCESS;
12128 }
12129 else
12130 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12131
12132 return rcStrict;
12133}
12134
12135
12136/**
12137 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12138 */
12139HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12140{
12141 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12142 /** @todo The guest has likely hit a contended spinlock. We might want to
12143 * poke a schedule different guest VCPU. */
12144 return VINF_EM_RAW_INTERRUPT;
12145}
12146
12147
12148/**
12149 * VM-exit handler for when the TPR value is lowered below the specified
12150 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12151 */
12152HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12153{
12154 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12155 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
12156
12157 /*
12158 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12159 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12160 */
12161 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12162 return VINF_SUCCESS;
12163}
12164
12165
12166/**
12167 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12168 * VM-exit.
12169 *
12170 * @retval VINF_SUCCESS when guest execution can continue.
12171 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12172 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12173 * interpreter.
12174 */
12175HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12176{
12177 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12178 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12179
12180 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12181 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12182 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12183 AssertRCReturn(rc, rc);
12184
12185 VBOXSTRICTRC rcStrict;
12186 PVM pVM = pVCpu->CTX_SUFF(pVM);
12187 RTGCUINTPTR const uExitQual = pVmxTransient->uExitQual;
12188 uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
12189 switch (uAccessType)
12190 {
12191 case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
12192 {
12193 uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
12194 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
12195 VMX_EXIT_QUAL_CRX_GENREG(uExitQual));
12196 AssertMsg( rcStrict == VINF_SUCCESS
12197 || rcStrict == VINF_IEM_RAISED_XCPT
12198 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12199
12200 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
12201 {
12202 case 0:
12203 {
12204 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12205 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12206 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
12207 Log4Func(("CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
12208
12209 /*
12210 * This is a kludge for handling switches back to real mode when we try to use
12211 * V86 mode to run real mode code directly. Problem is that V86 mode cannot
12212 * deal with special selector values, so we have to return to ring-3 and run
12213 * there till the selector values are V86 mode compatible.
12214 *
12215 * Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
12216 * latter is an alias for VINF_IEM_RAISED_XCPT which is converted to VINF_SUCCESs
12217 * at the end of this function.
12218 */
12219 if ( rc == VINF_SUCCESS
12220 && !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
12221 && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
12222 && (uOldCr0 & X86_CR0_PE)
12223 && !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
12224 {
12225 /** @todo check selectors rather than returning all the time. */
12226 Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
12227 rcStrict = VINF_EM_RESCHEDULE_REM;
12228 }
12229 break;
12230 }
12231
12232 case 2:
12233 {
12234 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
12235 /* Nothing to do here, CR2 it's not part of the VMCS. */
12236 break;
12237 }
12238
12239 case 3:
12240 {
12241 Assert( !pVM->hm.s.fNestedPaging
12242 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
12243 || pVCpu->hm.s.fUsingDebugLoop);
12244 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
12245 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12246 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR3);
12247 Log4Func(("CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
12248 break;
12249 }
12250
12251 case 4:
12252 {
12253 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
12254 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12255 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR4);
12256 Log4Func(("CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
12257 pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12258 break;
12259 }
12260
12261 case 8:
12262 {
12263 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
12264 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
12265 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12266 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_APIC_TPR);
12267 break;
12268 }
12269 default:
12270 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual)));
12271 break;
12272 }
12273 break;
12274 }
12275
12276 case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
12277 {
12278 Assert( !pVM->hm.s.fNestedPaging
12279 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
12280 || pVCpu->hm.s.fUsingDebugLoop
12281 || VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 3);
12282 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12283 Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 8
12284 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
12285
12286 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_GENREG(uExitQual),
12287 VMX_EXIT_QUAL_CRX_REGISTER(uExitQual));
12288 AssertMsg( rcStrict == VINF_SUCCESS
12289 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12290#ifdef VBOX_WITH_STATISTICS
12291 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
12292 {
12293 case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
12294 case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
12295 case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
12296 case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
12297 case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
12298 }
12299#endif
12300 Log4Func(("CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
12301 VBOXSTRICTRC_VAL(rcStrict)));
12302 if (VMX_EXIT_QUAL_CRX_GENREG(uExitQual) == X86_GREG_xSP)
12303 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RSP);
12304 else
12305 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12306 break;
12307 }
12308
12309 case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12310 {
12311 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12312 AssertMsg( rcStrict == VINF_SUCCESS
12313 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12314
12315 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12316 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12317 Log4Func(("CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12318 break;
12319 }
12320
12321 case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12322 {
12323 /* Note! LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here. */
12324 rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
12325 AssertRCReturn(rc, rc);
12326 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual),
12327 pVmxTransient->uGuestLinearAddr);
12328 AssertMsg( rcStrict == VINF_SUCCESS
12329 || rcStrict == VINF_IEM_RAISED_XCPT
12330 , ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12331
12332 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12333 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12334 Log4Func(("LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12335 break;
12336 }
12337
12338 default:
12339 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12340 VERR_VMX_UNEXPECTED_EXCEPTION);
12341 }
12342
12343 Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS))
12344 == (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS));
12345 if (rcStrict == VINF_IEM_RAISED_XCPT)
12346 {
12347 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
12348 rcStrict = VINF_SUCCESS;
12349 }
12350
12351 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12352 NOREF(pVM);
12353 return rcStrict;
12354}
12355
12356
12357/**
12358 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12359 * VM-exit.
12360 */
12361HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12362{
12363 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12364 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12365
12366 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12367 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12368 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12369 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_EFER);
12370 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12371 AssertRCReturn(rc, rc);
12372
12373 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12374 uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
12375 uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQual);
12376 bool fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12377 bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
12378 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
12379 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12380 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12381
12382 /*
12383 * Update exit history to see if this exit can be optimized.
12384 */
12385 VBOXSTRICTRC rcStrict;
12386 PCEMEXITREC pExitRec = NULL;
12387 if ( !fGstStepping
12388 && !fDbgStepping)
12389 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12390 !fIOString
12391 ? !fIOWrite
12392 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12393 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12394 : !fIOWrite
12395 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12396 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12397 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12398 if (!pExitRec)
12399 {
12400 /* I/O operation lookup arrays. */
12401 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12402 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12403 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12404 uint32_t const cbInstr = pVmxTransient->cbInstr;
12405 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12406 PVM pVM = pVCpu->CTX_SUFF(pVM);
12407 if (fIOString)
12408 {
12409 /*
12410 * INS/OUTS - I/O String instruction.
12411 *
12412 * Use instruction-information if available, otherwise fall back on
12413 * interpreting the instruction.
12414 */
12415 Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12416 AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
12417 bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
12418 if (fInsOutsInfo)
12419 {
12420 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12421 AssertRCReturn(rc2, rc2);
12422 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12423 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12424 IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12425 bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
12426 if (fIOWrite)
12427 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12428 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12429 else
12430 {
12431 /*
12432 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12433 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12434 * See Intel Instruction spec. for "INS".
12435 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12436 */
12437 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12438 }
12439 }
12440 else
12441 rcStrict = IEMExecOne(pVCpu);
12442
12443 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12444 fUpdateRipAlready = true;
12445 }
12446 else
12447 {
12448 /*
12449 * IN/OUT - I/O instruction.
12450 */
12451 Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12452 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12453 Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
12454 if (fIOWrite)
12455 {
12456 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
12457 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12458 if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
12459 && !pCtx->eflags.Bits.u1TF)
12460 rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
12461 }
12462 else
12463 {
12464 uint32_t u32Result = 0;
12465 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12466 if (IOM_SUCCESS(rcStrict))
12467 {
12468 /* Save result of I/O IN instr. in AL/AX/EAX. */
12469 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12470 }
12471 if ( rcStrict == VINF_IOM_R3_IOPORT_READ
12472 && !pCtx->eflags.Bits.u1TF)
12473 rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
12474 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12475 }
12476 }
12477
12478 if (IOM_SUCCESS(rcStrict))
12479 {
12480 if (!fUpdateRipAlready)
12481 {
12482 hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
12483 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12484 }
12485
12486 /*
12487 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12488 * while booting Fedora 17 64-bit guest.
12489 *
12490 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12491 */
12492 if (fIOString)
12493 {
12494 /** @todo Single-step for INS/OUTS with REP prefix? */
12495 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12496 }
12497 else if ( !fDbgStepping
12498 && fGstStepping)
12499 {
12500 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12501 AssertRCReturn(rc, rc);
12502 }
12503
12504 /*
12505 * If any I/O breakpoints are armed, we need to check if one triggered
12506 * and take appropriate action.
12507 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12508 */
12509 rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_DR7);
12510 AssertRCReturn(rc, rc);
12511
12512 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12513 * execution engines about whether hyper BPs and such are pending. */
12514 uint32_t const uDr7 = pCtx->dr[7];
12515 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12516 && X86_DR7_ANY_RW_IO(uDr7)
12517 && (pCtx->cr4 & X86_CR4_DE))
12518 || DBGFBpIsHwIoArmed(pVM)))
12519 {
12520 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12521
12522 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12523 VMMRZCallRing3Disable(pVCpu);
12524 HM_DISABLE_PREEMPT(pVCpu);
12525
12526 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12527
12528 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
12529 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12530 {
12531 /* Raise #DB. */
12532 if (fIsGuestDbgActive)
12533 ASMSetDR6(pCtx->dr[6]);
12534 if (pCtx->dr[7] != uDr7)
12535 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR7;
12536
12537 hmR0VmxSetPendingXcptDB(pVCpu);
12538 }
12539 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12540 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12541 else if ( rcStrict2 != VINF_SUCCESS
12542 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12543 rcStrict = rcStrict2;
12544 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12545
12546 HM_RESTORE_PREEMPT();
12547 VMMRZCallRing3Enable(pVCpu);
12548 }
12549 }
12550
12551#ifdef VBOX_STRICT
12552 if ( rcStrict == VINF_IOM_R3_IOPORT_READ
12553 || rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
12554 Assert(!fIOWrite);
12555 else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
12556 || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
12557 || rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
12558 Assert(fIOWrite);
12559 else
12560 {
12561# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12562 * statuses, that the VMM device and some others may return. See
12563 * IOM_SUCCESS() for guidance. */
12564 AssertMsg( RT_FAILURE(rcStrict)
12565 || rcStrict == VINF_SUCCESS
12566 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12567 || rcStrict == VINF_EM_DBG_BREAKPOINT
12568 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12569 || rcStrict == VINF_EM_RAW_TO_R3
12570 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12571# endif
12572 }
12573#endif
12574 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12575 }
12576 else
12577 {
12578 /*
12579 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12580 */
12581 int rc2 = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12582 AssertRCReturn(rc2, rc2);
12583 STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12584 : fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12585 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12586 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12587 VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
12588 fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12589
12590 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12591 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12592
12593 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12594 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12595 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12596 }
12597 return rcStrict;
12598}
12599
12600
12601/**
12602 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12603 * VM-exit.
12604 */
12605HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12606{
12607 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12608
12609 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12610 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12611 AssertRCReturn(rc, rc);
12612 if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12613 {
12614 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12615 AssertRCReturn(rc, rc);
12616 if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
12617 {
12618 uint32_t uErrCode;
12619 RTGCUINTPTR GCPtrFaultAddress;
12620 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12621 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12622 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
12623 if (fErrorCodeValid)
12624 {
12625 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12626 AssertRCReturn(rc, rc);
12627 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12628 }
12629 else
12630 uErrCode = 0;
12631
12632 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12633 && uVector == X86_XCPT_PF)
12634 GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
12635 else
12636 GCPtrFaultAddress = 0;
12637
12638 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12639 AssertRCReturn(rc, rc);
12640
12641 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12642 pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
12643
12644 Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
12645 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12646 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12647 }
12648 }
12649
12650 /* Fall back to the interpreter to emulate the task-switch. */
12651 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12652 return VERR_EM_INTERPRETER;
12653}
12654
12655
12656/**
12657 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12658 */
12659HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12660{
12661 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12662 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_MONITOR_TRAP_FLAG);
12663 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
12664 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12665 AssertRCReturn(rc, rc);
12666 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12667 return VINF_EM_DBG_STEPPED;
12668}
12669
12670
12671/**
12672 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12673 */
12674HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12675{
12676 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12677
12678 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12679
12680 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12681 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12682 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12683 {
12684 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12685 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12686 {
12687 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12688 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12689 }
12690 }
12691 else
12692 {
12693 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12694 rcStrict1 = VINF_SUCCESS;
12695 return rcStrict1;
12696 }
12697
12698 /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12699 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12700 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12701 AssertRCReturn(rc, rc);
12702
12703 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12704 uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
12705 VBOXSTRICTRC rcStrict2;
12706 switch (uAccessType)
12707 {
12708 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12709 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12710 {
12711 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
12712 || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
12713 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12714
12715 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12716 GCPhys &= PAGE_BASE_GC_MASK;
12717 GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
12718 PVM pVM = pVCpu->CTX_SUFF(pVM);
12719 Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12720 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
12721
12722 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12723 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12724 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12725 CPUMCTX2CORE(pCtx), GCPhys);
12726 Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12727 if ( rcStrict2 == VINF_SUCCESS
12728 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12729 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12730 {
12731 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12732 | HM_CHANGED_GUEST_APIC_TPR);
12733 rcStrict2 = VINF_SUCCESS;
12734 }
12735 break;
12736 }
12737
12738 default:
12739 Log4Func(("uAccessType=%#x\n", uAccessType));
12740 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12741 break;
12742 }
12743
12744 if (rcStrict2 != VINF_SUCCESS)
12745 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12746 return rcStrict2;
12747}
12748
12749
12750/**
12751 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12752 * VM-exit.
12753 */
12754HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12755{
12756 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12757
12758 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12759 if (pVmxTransient->fWasGuestDebugStateActive)
12760 {
12761 AssertMsgFailed(("Unexpected MOV DRx exit\n"));
12762 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12763 }
12764
12765 if ( !pVCpu->hm.s.fSingleInstruction
12766 && !pVmxTransient->fWasHyperDebugStateActive)
12767 {
12768 Assert(!DBGFIsStepping(pVCpu));
12769 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12770
12771 /* Don't intercept MOV DRx any more. */
12772 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
12773 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12774 AssertRCReturn(rc, rc);
12775
12776 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12777 VMMRZCallRing3Disable(pVCpu);
12778 HM_DISABLE_PREEMPT(pVCpu);
12779
12780 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12781 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12782 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12783
12784 HM_RESTORE_PREEMPT();
12785 VMMRZCallRing3Enable(pVCpu);
12786
12787#ifdef VBOX_WITH_STATISTICS
12788 rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12789 AssertRCReturn(rc, rc);
12790 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12791 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12792 else
12793 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12794#endif
12795 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12796 return VINF_SUCCESS;
12797 }
12798
12799 /*
12800 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12801 * Update the segment registers and DR7 from the CPU.
12802 */
12803 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12804 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12805 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_DR7);
12806 AssertRCReturn(rc, rc);
12807 Log4Func(("CS:RIP=%04x:%08RX64\n", pCtx->cs.Sel, pCtx->rip));
12808
12809 PVM pVM = pVCpu->CTX_SUFF(pVM);
12810 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12811 {
12812 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12813 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
12814 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
12815 if (RT_SUCCESS(rc))
12816 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12817 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12818 }
12819 else
12820 {
12821 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12822 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
12823 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
12824 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12825 }
12826
12827 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12828 if (RT_SUCCESS(rc))
12829 {
12830 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12831 AssertRCReturn(rc2, rc2);
12832 return VINF_SUCCESS;
12833 }
12834 return rc;
12835}
12836
12837
12838/**
12839 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12840 * Conditional VM-exit.
12841 */
12842HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12843{
12844 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12845 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12846
12847 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12848 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12849 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12850 {
12851 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12852 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12853 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12854 {
12855 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12856 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12857 }
12858 }
12859 else
12860 {
12861 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12862 rcStrict1 = VINF_SUCCESS;
12863 return rcStrict1;
12864 }
12865
12866 /*
12867 * Get sufficent state and update the exit history entry.
12868 */
12869 RTGCPHYS GCPhys;
12870 int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
12871 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12872 AssertRCReturn(rc, rc);
12873
12874 VBOXSTRICTRC rcStrict;
12875 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12876 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
12877 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12878 if (!pExitRec)
12879 {
12880 /*
12881 * If we succeed, resume guest execution.
12882 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12883 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12884 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12885 * weird case. See @bugref{6043}.
12886 */
12887 PVM pVM = pVCpu->CTX_SUFF(pVM);
12888 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12889 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
12890 Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12891 if ( rcStrict == VINF_SUCCESS
12892 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12893 || rcStrict == VERR_PAGE_NOT_PRESENT)
12894 {
12895 /* Successfully handled MMIO operation. */
12896 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12897 | HM_CHANGED_GUEST_APIC_TPR);
12898 rcStrict = VINF_SUCCESS;
12899 }
12900 }
12901 else
12902 {
12903 /*
12904 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12905 */
12906 int rc2 = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12907 AssertRCReturn(rc2, rc2);
12908
12909 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12910 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12911
12912 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12913 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12914
12915 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12916 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12917 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12918 }
12919 return VBOXSTRICTRC_TODO(rcStrict);
12920}
12921
12922
12923/**
12924 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12925 * VM-exit.
12926 */
12927HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12928{
12929 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12930 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12931
12932 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12933 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12934 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12935 {
12936 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12937 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12938 Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12939 }
12940 else
12941 {
12942 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12943 rcStrict1 = VINF_SUCCESS;
12944 return rcStrict1;
12945 }
12946
12947 RTGCPHYS GCPhys;
12948 int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
12949 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12950 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12951 AssertRCReturn(rc, rc);
12952
12953 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12954 AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
12955
12956 RTGCUINT uErrorCode = 0;
12957 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12958 uErrorCode |= X86_TRAP_PF_ID;
12959 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12960 uErrorCode |= X86_TRAP_PF_RW;
12961 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12962 uErrorCode |= X86_TRAP_PF_P;
12963
12964 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12965
12966
12967 /* Handle the pagefault trap for the nested shadow table. */
12968 PVM pVM = pVCpu->CTX_SUFF(pVM);
12969 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12970
12971 Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
12972 pCtx->cs.Sel, pCtx->rip));
12973
12974 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
12975 TRPMResetTrap(pVCpu);
12976
12977 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12978 if ( rcStrict2 == VINF_SUCCESS
12979 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12980 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12981 {
12982 /* Successfully synced our nested page tables. */
12983 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12984 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
12985 return VINF_SUCCESS;
12986 }
12987
12988 Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12989 return rcStrict2;
12990}
12991
12992/** @} */
12993
12994/** @name VM-exit exception handlers.
12995 * @{
12996 */
12997/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12998/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12999/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13000
13001/**
13002 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
13003 */
13004static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13005{
13006 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13007 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13008
13009 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0);
13010 AssertRCReturn(rc, rc);
13011
13012 if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
13013 {
13014 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13015 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13016
13017 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13018 * provides VM-exit instruction length. If this causes problem later,
13019 * disassemble the instruction like it's done on AMD-V. */
13020 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13021 AssertRCReturn(rc2, rc2);
13022 return rc;
13023 }
13024
13025 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13026 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13027 return rc;
13028}
13029
13030
13031/**
13032 * VM-exit exception handler for \#BP (Breakpoint exception).
13033 */
13034static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13035{
13036 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13037 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13038
13039 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13040 AssertRCReturn(rc, rc);
13041
13042 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13043 rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13044 if (rc == VINF_EM_RAW_GUEST_TRAP)
13045 {
13046 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13047 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13048 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13049 AssertRCReturn(rc, rc);
13050
13051 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13052 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13053 }
13054
13055 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
13056 return rc;
13057}
13058
13059
13060/**
13061 * VM-exit exception handler for \#AC (alignment check exception).
13062 */
13063static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13064{
13065 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13066
13067 /*
13068 * Re-inject it. We'll detect any nesting before getting here.
13069 */
13070 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13071 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13072 AssertRCReturn(rc, rc);
13073 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13074
13075 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13076 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13077 return VINF_SUCCESS;
13078}
13079
13080
13081/**
13082 * VM-exit exception handler for \#DB (Debug exception).
13083 */
13084static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13085{
13086 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13087 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13088
13089 /*
13090 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13091 * for processing.
13092 */
13093 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13094
13095 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13096 uint64_t uDR6 = X86_DR6_INIT_VAL;
13097 uDR6 |= (pVmxTransient->uExitQual & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
13098
13099 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13100 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13101 Log6Func(("rc=%Rrc\n", rc));
13102 if (rc == VINF_EM_RAW_GUEST_TRAP)
13103 {
13104 /*
13105 * The exception was for the guest. Update DR6, DR7.GD and
13106 * IA32_DEBUGCTL.LBR before forwarding it.
13107 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13108 */
13109 VMMRZCallRing3Disable(pVCpu);
13110 HM_DISABLE_PREEMPT(pVCpu);
13111
13112 pCtx->dr[6] &= ~X86_DR6_B_MASK;
13113 pCtx->dr[6] |= uDR6;
13114 if (CPUMIsGuestDebugStateActive(pVCpu))
13115 ASMSetDR6(pCtx->dr[6]);
13116
13117 HM_RESTORE_PREEMPT();
13118 VMMRZCallRing3Enable(pVCpu);
13119
13120 rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_DR7);
13121 AssertRCReturn(rc, rc);
13122
13123 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13124 pCtx->dr[7] &= ~X86_DR7_GD;
13125
13126 /* Paranoia. */
13127 pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13128 pCtx->dr[7] |= X86_DR7_RA1_MASK;
13129
13130 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
13131 AssertRCReturn(rc, rc);
13132
13133 /*
13134 * Raise #DB in the guest.
13135 *
13136 * It is important to reflect exactly what the VM-exit gave us (preserving the
13137 * interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
13138 * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
13139 * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
13140 *
13141 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
13142 * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
13143 */
13144 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13145 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13146 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13147 AssertRCReturn(rc, rc);
13148 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13149 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13150 return VINF_SUCCESS;
13151 }
13152
13153 /*
13154 * Not a guest trap, must be a hypervisor related debug event then.
13155 * Update DR6 in case someone is interested in it.
13156 */
13157 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13158 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13159 CPUMSetHyperDR6(pVCpu, uDR6);
13160
13161 return rc;
13162}
13163
13164
13165/**
13166 * VM-exit exception handler for \#GP (General-protection exception).
13167 *
13168 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13169 */
13170static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13171{
13172 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13173 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13174
13175 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13176 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13177 { /* likely */ }
13178 else
13179 {
13180#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13181 Assert(pVCpu->hm.s.fUsingDebugLoop);
13182#endif
13183 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13184 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13185 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13186 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13187 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13188 AssertRCReturn(rc, rc);
13189 Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pCtx->cs.Sel, pCtx->rip,
13190 pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
13191 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13192 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13193 return rc;
13194 }
13195
13196 Assert(CPUMIsGuestInRealModeEx(pCtx));
13197 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13198
13199 int rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13200 AssertRCReturn(rc, rc);
13201
13202 VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
13203 if (rcStrict == VINF_SUCCESS)
13204 {
13205 if (!CPUMIsGuestInRealModeEx(pCtx))
13206 {
13207 /*
13208 * The guest is no longer in real-mode, check if we can continue executing the
13209 * guest using hardware-assisted VMX. Otherwise, fall back to emulation.
13210 */
13211 if (HMVmxCanExecuteGuest(pVCpu, pCtx))
13212 {
13213 Log4Func(("Mode changed but guest still suitable for executing using VT-x\n"));
13214 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
13215 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13216 }
13217 else
13218 {
13219 Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
13220 rcStrict = VINF_EM_RESCHEDULE;
13221 }
13222 }
13223 else
13224 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13225 }
13226 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13227 {
13228 rcStrict = VINF_SUCCESS;
13229 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13230 }
13231 return VBOXSTRICTRC_VAL(rcStrict);
13232}
13233
13234
13235/**
13236 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13237 * the exception reported in the VMX transient structure back into the VM.
13238 *
13239 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13240 * up-to-date.
13241 */
13242static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13243{
13244 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13245#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13246 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13247 ("uVector=%#x u32XcptBitmap=%#X32\n",
13248 VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13249#endif
13250
13251 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13252 hmR0VmxCheckExitDueToEventDelivery(). */
13253 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13254 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13255 AssertRCReturn(rc, rc);
13256 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13257
13258#ifdef DEBUG_ramshankar
13259 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
13260 uint8_t uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13261 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13262#endif
13263
13264 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13265 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13266 return VINF_SUCCESS;
13267}
13268
13269
13270/**
13271 * VM-exit exception handler for \#PF (Page-fault exception).
13272 */
13273static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13274{
13275 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13276 PVM pVM = pVCpu->CTX_SUFF(pVM);
13277 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13278 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13279 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13280 AssertRCReturn(rc, rc);
13281
13282 if (!pVM->hm.s.fNestedPaging)
13283 { /* likely */ }
13284 else
13285 {
13286#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13287 Assert(pVCpu->hm.s.fUsingDebugLoop);
13288#endif
13289 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13290 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13291 {
13292 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13293 pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
13294 }
13295 else
13296 {
13297 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13298 hmR0VmxSetPendingXcptDF(pVCpu);
13299 Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13300 }
13301 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13302 return rc;
13303 }
13304
13305 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13306 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13307 if (pVmxTransient->fVectoringPF)
13308 {
13309 Assert(pVCpu->hm.s.Event.fPending);
13310 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13311 }
13312
13313 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13314 rc = HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13315 AssertRCReturn(rc, rc);
13316
13317 Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
13318 pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
13319
13320 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13321 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
13322
13323 Log4Func(("#PF: rc=%Rrc\n", rc));
13324 if (rc == VINF_SUCCESS)
13325 {
13326 /*
13327 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13328 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13329 */
13330 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13331 TRPMResetTrap(pVCpu);
13332 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13333 return rc;
13334 }
13335
13336 if (rc == VINF_EM_RAW_GUEST_TRAP)
13337 {
13338 if (!pVmxTransient->fVectoringDoublePF)
13339 {
13340 /* It's a guest page fault and needs to be reflected to the guest. */
13341 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13342 TRPMResetTrap(pVCpu);
13343 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13344 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13345 uGstErrorCode, pVmxTransient->uExitQual);
13346 }
13347 else
13348 {
13349 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13350 TRPMResetTrap(pVCpu);
13351 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13352 hmR0VmxSetPendingXcptDF(pVCpu);
13353 Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13354 }
13355
13356 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13357 return VINF_SUCCESS;
13358 }
13359
13360 TRPMResetTrap(pVCpu);
13361 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13362 return rc;
13363}
13364
13365/** @} */
13366
13367#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13368/** @name Nested-guest VM-exit handlers.
13369 * @{
13370 */
13371/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13372/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= Nested-guest VM-exit handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13373/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13374
13375/**
13376 * VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
13377 */
13378HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13379{
13380 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13381
13382 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13383 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13384 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13385 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13386 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13387 AssertRCReturn(rc, rc);
13388
13389 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13390
13391 VMXVEXITINFO ExitInfo;
13392 RT_ZERO(ExitInfo);
13393 ExitInfo.uReason = pVmxTransient->uExitReason;
13394 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13395 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13396 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13397 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13398
13399 VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
13400 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13401 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13402 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13403 {
13404 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13405 rcStrict = VINF_SUCCESS;
13406 }
13407 return rcStrict;
13408}
13409
13410
13411/**
13412 * VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
13413 */
13414HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13415{
13416 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13417
13418 /** @todo NSTVMX: Vmlaunch. */
13419 hmR0VmxSetPendingXcptUD(pVCpu);
13420 return VINF_SUCCESS;
13421}
13422
13423
13424/**
13425 * VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
13426 */
13427HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13428{
13429 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13430
13431 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13432 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13433 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13434 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13435 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13436 AssertRCReturn(rc, rc);
13437
13438 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13439
13440 VMXVEXITINFO ExitInfo;
13441 RT_ZERO(ExitInfo);
13442 ExitInfo.uReason = pVmxTransient->uExitReason;
13443 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13444 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13445 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13446 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13447
13448 VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
13449 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13450 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13451 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13452 {
13453 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13454 rcStrict = VINF_SUCCESS;
13455 }
13456 return rcStrict;
13457}
13458
13459
13460/**
13461 * VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
13462 */
13463HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13464{
13465 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13466
13467 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13468 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13469 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13470 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13471 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13472 AssertRCReturn(rc, rc);
13473
13474 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13475
13476 VMXVEXITINFO ExitInfo;
13477 RT_ZERO(ExitInfo);
13478 ExitInfo.uReason = pVmxTransient->uExitReason;
13479 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13480 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13481 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13482 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
13483
13484 VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
13485 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13486 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13487 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13488 {
13489 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13490 rcStrict = VINF_SUCCESS;
13491 }
13492 return rcStrict;
13493}
13494
13495
13496/**
13497 * VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Unconditional VM-exit.
13498 */
13499HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13500{
13501 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13502
13503 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13504 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13505 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13506 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13507 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13508 AssertRCReturn(rc, rc);
13509
13510 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13511
13512 VMXVEXITINFO ExitInfo;
13513 RT_ZERO(ExitInfo);
13514 ExitInfo.uReason = pVmxTransient->uExitReason;
13515 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13516 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13517 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13518 if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
13519 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
13520
13521 VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
13522 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13523 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13524 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13525 {
13526 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13527 rcStrict = VINF_SUCCESS;
13528 }
13529 return rcStrict;
13530}
13531
13532
13533/**
13534 * VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
13535 */
13536HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13537{
13538 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13539
13540 /** @todo NSTVMX: Vmresume. */
13541 hmR0VmxSetPendingXcptUD(pVCpu);
13542 return VINF_SUCCESS;
13543}
13544
13545
13546/**
13547 * VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Unconditional VM-exit.
13548 */
13549HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13550{
13551 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13552
13553 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13554 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13555 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13556 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13557 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13558 AssertRCReturn(rc, rc);
13559
13560 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13561
13562 VMXVEXITINFO ExitInfo;
13563 RT_ZERO(ExitInfo);
13564 ExitInfo.uReason = pVmxTransient->uExitReason;
13565 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13566 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13567 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13568 if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
13569 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13570
13571 VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
13572 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13573 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13574 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13575 {
13576 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13577 rcStrict = VINF_SUCCESS;
13578 }
13579 return rcStrict;
13580}
13581
13582
13583/**
13584 * VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
13585 */
13586HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13587{
13588 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13589
13590 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13591 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13592 AssertRCReturn(rc, rc);
13593
13594 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13595
13596 VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
13597 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13598 {
13599 /* VMXOFF on success changes the internal hwvirt state but not anything that's visible to the guest. */
13600 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
13601 }
13602 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13603 {
13604 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13605 rcStrict = VINF_SUCCESS;
13606 }
13607 return rcStrict;
13608}
13609
13610
13611/**
13612 * VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
13613 */
13614HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13615{
13616 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13617
13618 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13619 rc |= HMVMX_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13620 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13621 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13622 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13623 AssertRCReturn(rc, rc);
13624
13625 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13626
13627 VMXVEXITINFO ExitInfo;
13628 RT_ZERO(ExitInfo);
13629 ExitInfo.uReason = pVmxTransient->uExitReason;
13630 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13631 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13632 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13633 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13634
13635 VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
13636 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13637 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13638 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13639 {
13640 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13641 rcStrict = VINF_SUCCESS;
13642 }
13643 return rcStrict;
13644}
13645
13646/** @} */
13647#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13648
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette