VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 79345

Last change on this file since 79345 was 79345, checked in by vboxsync, 5 years ago

VMM/HM: Nested VMX: bugref:9180 Implemented VMCS shadowing.

  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 730.7 KB
Line 
1/* $Id: HMVMXR0.cpp 79345 2019-06-26 09:09:46Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2019 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/x86.h>
25#include <iprt/asm-amd64-x86.h>
26#include <iprt/thread.h>
27#include <iprt/mem.h>
28
29#include <VBox/vmm/pdmapi.h>
30#include <VBox/vmm/dbgf.h>
31#include <VBox/vmm/iem.h>
32#include <VBox/vmm/iom.h>
33#include <VBox/vmm/selm.h>
34#include <VBox/vmm/tm.h>
35#include <VBox/vmm/em.h>
36#include <VBox/vmm/gim.h>
37#include <VBox/vmm/apic.h>
38#ifdef VBOX_WITH_REM
39# include <VBox/vmm/rem.h>
40#endif
41#include "HMInternal.h"
42#include <VBox/vmm/vm.h>
43#include <VBox/vmm/hmvmxinline.h>
44#include "HMVMXR0.h"
45#include "dtrace/VBoxVMM.h"
46
47#ifdef DEBUG_ramshankar
48# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
49# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
50# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
51# define HMVMX_ALWAYS_CHECK_GUEST_STATE
52# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
53# define HMVMX_ALWAYS_TRAP_PF
54# define HMVMX_ALWAYS_FLUSH_TLB
55# define HMVMX_ALWAYS_SWAP_EFER
56#endif
57
58
59/*********************************************************************************************************************************
60* Defined Constants And Macros *
61*********************************************************************************************************************************/
62/** Use the function table. */
63#define HMVMX_USE_FUNCTION_TABLE
64
65/** Determine which tagged-TLB flush handler to use. */
66#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
67#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
68#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
69#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
70
71/** @name HMVMX_READ_XXX
72 * Flags to skip redundant reads of some common VMCS fields that are not part of
73 * the guest-CPU or VCPU state but are needed while handling VM-exits.
74 */
75#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
76#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
77#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
78#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
79#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
80#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
81#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
82#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
83/** @} */
84
85/**
86 * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
87 * guest using hardware-assisted VMX.
88 *
89 * This excludes state like GPRs (other than RSP) which are always are
90 * swapped and restored across the world-switch and also registers like EFER,
91 * MSR which cannot be modified by the guest without causing a VM-exit.
92 */
93#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
94 | CPUMCTX_EXTRN_RFLAGS \
95 | CPUMCTX_EXTRN_RSP \
96 | CPUMCTX_EXTRN_SREG_MASK \
97 | CPUMCTX_EXTRN_TABLE_MASK \
98 | CPUMCTX_EXTRN_KERNEL_GS_BASE \
99 | CPUMCTX_EXTRN_SYSCALL_MSRS \
100 | CPUMCTX_EXTRN_SYSENTER_MSRS \
101 | CPUMCTX_EXTRN_TSC_AUX \
102 | CPUMCTX_EXTRN_OTHER_MSRS \
103 | CPUMCTX_EXTRN_CR0 \
104 | CPUMCTX_EXTRN_CR3 \
105 | CPUMCTX_EXTRN_CR4 \
106 | CPUMCTX_EXTRN_DR7 \
107 | CPUMCTX_EXTRN_HWVIRT \
108 | CPUMCTX_EXTRN_HM_VMX_MASK)
109
110/**
111 * Exception bitmap mask for real-mode guests (real-on-v86).
112 *
113 * We need to intercept all exceptions manually except:
114 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
115 * due to bugs in Intel CPUs.
116 * - \#PF need not be intercepted even in real-mode if we have nested paging
117 * support.
118 */
119#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
120 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
121 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
122 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
123 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
124 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
125 | RT_BIT(X86_XCPT_XF))
126
127/** Maximum VM-instruction error number. */
128#define HMVMX_INSTR_ERROR_MAX 28
129
130/** Profiling macro. */
131#ifdef HM_PROFILE_EXIT_DISPATCH
132# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
133# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
134#else
135# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
136# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
137#endif
138
139/** Assert that preemption is disabled or covered by thread-context hooks. */
140#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
141 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
142
143/** Assert that we haven't migrated CPUs when thread-context hooks are not
144 * used. */
145#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
146 || (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
147 ("Illegal migration! Entered on CPU %u Current %u\n", \
148 (a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
149
150/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
151 * context. */
152#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
153 ("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
154 (a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
155
156/** Helper macro for VM-exit handlers called unexpectedly. */
157#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_HmError) \
158 do { \
159 (a_pVCpu)->hm.s.u32HMError = (a_HmError); \
160 return VERR_VMX_UNEXPECTED_EXIT; \
161 } while (0)
162
163#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
164/** Macro that does the necessary privilege checks and intercepted VM-exits for
165 * guests that attempted to execute a VMX instruction. */
166# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
167 do \
168 { \
169 VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
170 if (rcStrictTmp == VINF_SUCCESS) \
171 { /* likely */ } \
172 else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
173 { \
174 Assert((a_pVCpu)->hm.s.Event.fPending); \
175 Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
176 return VINF_SUCCESS; \
177 } \
178 else \
179 { \
180 int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
181 AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
182 } \
183 } while (0)
184
185/** Macro that decodes a memory operand for an instruction VM-exit. */
186# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
187 do \
188 { \
189 VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
190 (a_pGCPtrEffAddr)); \
191 if (rcStrictTmp == VINF_SUCCESS) \
192 { /* likely */ } \
193 else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
194 { \
195 uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
196 Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
197 NOREF(uXcptTmp); \
198 return VINF_SUCCESS; \
199 } \
200 else \
201 { \
202 Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
203 return rcStrictTmp; \
204 } \
205 } while (0)
206
207#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
208
209
210/*********************************************************************************************************************************
211* Structures and Typedefs *
212*********************************************************************************************************************************/
213/**
214 * VMX transient state.
215 *
216 * A state structure for holding miscellaneous information across
217 * VMX non-root operation and restored after the transition.
218 */
219typedef struct VMXTRANSIENT
220{
221 /** The host's rflags/eflags. */
222 RTCCUINTREG fEFlags;
223#if HC_ARCH_BITS == 32
224 uint32_t u32Alignment0;
225#endif
226 /** The guest's TPR value used for TPR shadowing. */
227 uint8_t u8GuestTpr;
228 /** Alignment. */
229 uint8_t abAlignment0[7];
230
231 /** The basic VM-exit reason. */
232 uint16_t uExitReason;
233 /** Alignment. */
234 uint16_t u16Alignment0;
235 /** The VM-exit interruption error code. */
236 uint32_t uExitIntErrorCode;
237 /** The VM-exit exit code qualification. */
238 uint64_t uExitQual;
239 /** The Guest-linear address. */
240 uint64_t uGuestLinearAddr;
241
242 /** The VM-exit interruption-information field. */
243 uint32_t uExitIntInfo;
244 /** The VM-exit instruction-length field. */
245 uint32_t cbInstr;
246 /** The VM-exit instruction-information field. */
247 VMXEXITINSTRINFO ExitInstrInfo;
248 /** Whether the VM-entry failed or not. */
249 bool fVMEntryFailed;
250 /** Whether we are currently executing a nested-guest. */
251 bool fIsNestedGuest;
252 /** Alignment. */
253 uint8_t abAlignment1[2];
254
255 /** The VM-entry interruption-information field. */
256 uint32_t uEntryIntInfo;
257 /** The VM-entry exception error code field. */
258 uint32_t uEntryXcptErrorCode;
259 /** The VM-entry instruction length field. */
260 uint32_t cbEntryInstr;
261
262 /** IDT-vectoring information field. */
263 uint32_t uIdtVectoringInfo;
264 /** IDT-vectoring error code. */
265 uint32_t uIdtVectoringErrorCode;
266
267 /** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
268 uint32_t fVmcsFieldsRead;
269
270 /** Whether the guest debug state was active at the time of VM-exit. */
271 bool fWasGuestDebugStateActive;
272 /** Whether the hyper debug state was active at the time of VM-exit. */
273 bool fWasHyperDebugStateActive;
274 /** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
275 bool fUpdatedTscOffsettingAndPreemptTimer;
276 /** Whether the VM-exit was caused by a page-fault during delivery of a
277 * contributory exception or a page-fault. */
278 bool fVectoringDoublePF;
279 /** Whether the VM-exit was caused by a page-fault during delivery of an
280 * external interrupt or NMI. */
281 bool fVectoringPF;
282 bool afAlignment0[3];
283
284 /** The VMCS info. object. */
285 PVMXVMCSINFO pVmcsInfo;
286} VMXTRANSIENT;
287AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
288AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
289AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
290AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
291AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
292AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
293/** Pointer to VMX transient state. */
294typedef VMXTRANSIENT *PVMXTRANSIENT;
295/** Pointer to a const VMX transient state. */
296typedef const VMXTRANSIENT *PCVMXTRANSIENT;
297
298
299/**
300 * Memory operand read or write access.
301 */
302typedef enum VMXMEMACCESS
303{
304 VMXMEMACCESS_READ = 0,
305 VMXMEMACCESS_WRITE = 1
306} VMXMEMACCESS;
307
308/**
309 * VMX VM-exit handler.
310 *
311 * @returns Strict VBox status code (i.e. informational status codes too).
312 * @param pVCpu The cross context virtual CPU structure.
313 * @param pVmxTransient The VMX-transient structure.
314 */
315#ifndef HMVMX_USE_FUNCTION_TABLE
316typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
317#else
318typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
319/** Pointer to VM-exit handler. */
320typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
321#endif
322
323/**
324 * VMX VM-exit handler, non-strict status code.
325 *
326 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
327 *
328 * @returns VBox status code, no informational status code returned.
329 * @param pVCpu The cross context virtual CPU structure.
330 * @param pVmxTransient The VMX-transient structure.
331 *
332 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
333 * use of that status code will be replaced with VINF_EM_SOMETHING
334 * later when switching over to IEM.
335 */
336#ifndef HMVMX_USE_FUNCTION_TABLE
337typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
338#else
339typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
340#endif
341
342
343/*********************************************************************************************************************************
344* Internal Functions *
345*********************************************************************************************************************************/
346#ifndef HMVMX_USE_FUNCTION_TABLE
347DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
348# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
349# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
350#else
351# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
352# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
353#endif
354#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
355DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
356#endif
357
358static int hmR0VmxImportGuestState(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
359#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
360static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
361#endif
362
363/** @name VM-exit handlers.
364 * @{
365 */
366static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
367static FNVMXEXITHANDLER hmR0VmxExitExtInt;
368static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
369static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
370static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
371static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
372static FNVMXEXITHANDLER hmR0VmxExitCpuid;
373static FNVMXEXITHANDLER hmR0VmxExitGetsec;
374static FNVMXEXITHANDLER hmR0VmxExitHlt;
375static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
376static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
377static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
378static FNVMXEXITHANDLER hmR0VmxExitVmcall;
379#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
380static FNVMXEXITHANDLER hmR0VmxExitVmclear;
381static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
382static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
383static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
384static FNVMXEXITHANDLER hmR0VmxExitVmread;
385static FNVMXEXITHANDLER hmR0VmxExitVmresume;
386static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
387static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
388static FNVMXEXITHANDLER hmR0VmxExitVmxon;
389static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
390#endif
391static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
392static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
393static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
394static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
395static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
396static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
397static FNVMXEXITHANDLER hmR0VmxExitMwait;
398static FNVMXEXITHANDLER hmR0VmxExitMtf;
399static FNVMXEXITHANDLER hmR0VmxExitMonitor;
400static FNVMXEXITHANDLER hmR0VmxExitPause;
401static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
402static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
403static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
404static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
405static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
406static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
407static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
408static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
409static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
410static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
411static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
412static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUnexpected;
413/** @} */
414
415#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
416/** @name Nested-guest VM-exit handlers.
417 * @{
418 */
419static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmiNested;
420//static FNVMXEXITHANDLER hmR0VmxExitExtIntNested;
421static FNVMXEXITHANDLER hmR0VmxExitTripleFaultNested;
422static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindowNested;
423static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindowNested;
424static FNVMXEXITHANDLER hmR0VmxExitTaskSwitchNested;
425//static FNVMXEXITHANDLER hmR0VmxExitCpuid;
426//static FNVMXEXITHANDLER hmR0VmxExitGetsec;
427static FNVMXEXITHANDLER hmR0VmxExitHltNested;
428//static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
429static FNVMXEXITHANDLER hmR0VmxExitInvlpgNested;
430static FNVMXEXITHANDLER hmR0VmxExitRdpmcNested;
431//static FNVMXEXITHANDLER hmR0VmxExitVmcall;
432//static FNVMXEXITHANDLER hmR0VmxExitVmclear;
433//static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
434//static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
435//static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
436static FNVMXEXITHANDLER hmR0VmxExitVmreadVmwriteNested;
437//static FNVMXEXITHANDLER hmR0VmxExitVmresume;
438//static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
439//static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
440//static FNVMXEXITHANDLER hmR0VmxExitVmxon;
441//static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
442static FNVMXEXITHANDLER hmR0VmxExitRdtscNested;
443static FNVMXEXITHANDLER hmR0VmxExitMovCRxNested;
444static FNVMXEXITHANDLER hmR0VmxExitMovDRxNested;
445static FNVMXEXITHANDLER hmR0VmxExitIoInstrNested;
446static FNVMXEXITHANDLER hmR0VmxExitRdmsrNested;
447static FNVMXEXITHANDLER hmR0VmxExitWrmsrNested;
448static FNVMXEXITHANDLER hmR0VmxExitMwaitNested;
449static FNVMXEXITHANDLER hmR0VmxExitMtfNested;
450static FNVMXEXITHANDLER hmR0VmxExitMonitorNested;
451static FNVMXEXITHANDLER hmR0VmxExitPauseNested;
452static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThresholdNested;
453static FNVMXEXITHANDLER hmR0VmxExitApicAccessNested;
454static FNVMXEXITHANDLER hmR0VmxExitApicWriteNested;
455static FNVMXEXITHANDLER hmR0VmxExitVirtEoiNested;
456//static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
457//static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
458static FNVMXEXITHANDLER hmR0VmxExitRdtscpNested;
459//static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
460static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvdNested;
461//static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
462//static FNVMXEXITHANDLER hmR0VmxExitErrUnexpected;
463static FNVMXEXITHANDLER hmR0VmxExitInvpcidNested;
464//static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
465static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestStateNested;
466//static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUnexpected;
467static FNVMXEXITHANDLER hmR0VmxExitInstrNested;
468static FNVMXEXITHANDLER hmR0VmxExitInstrWithInfoNested;
469/** @} */
470#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
471
472/** @name Helpers for hardware exceptions VM-exit handlers.
473 * @{
474 */
475static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
476static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
477static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
478static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
479static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
480static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
481static VBOXSTRICTRC hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
482static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst);
483static VBOXSTRICTRC hmR0VmxExitClts(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr);
484static VBOXSTRICTRC hmR0VmxExitMovFromCrX(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg);
485static VBOXSTRICTRC hmR0VmxExitMovToCrX(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg);
486static VBOXSTRICTRC hmR0VmxExitHostNmi(PVMCPU pVCpu);
487/** @} */
488
489
490/*********************************************************************************************************************************
491* Global Variables *
492*********************************************************************************************************************************/
493#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
494/**
495 * Array of all VMCS fields.
496 * Any fields added to the VT-x spec. should be added here.
497 *
498 * Currently only used to derive shadow VMCS fields for hardware-assisted execution
499 * of nested-guests.
500 */
501static const uint32_t g_aVmcsFields[] =
502{
503 /* 16-bit control fields. */
504 VMX_VMCS16_VPID,
505 VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR,
506 VMX_VMCS16_EPTP_INDEX,
507
508 /* 16-bit guest-state fields. */
509 VMX_VMCS16_GUEST_ES_SEL,
510 VMX_VMCS16_GUEST_CS_SEL,
511 VMX_VMCS16_GUEST_SS_SEL,
512 VMX_VMCS16_GUEST_DS_SEL,
513 VMX_VMCS16_GUEST_FS_SEL,
514 VMX_VMCS16_GUEST_GS_SEL,
515 VMX_VMCS16_GUEST_LDTR_SEL,
516 VMX_VMCS16_GUEST_TR_SEL,
517 VMX_VMCS16_GUEST_INTR_STATUS,
518 VMX_VMCS16_GUEST_PML_INDEX,
519
520 /* 16-bits host-state fields. */
521 VMX_VMCS16_HOST_ES_SEL,
522 VMX_VMCS16_HOST_CS_SEL,
523 VMX_VMCS16_HOST_SS_SEL,
524 VMX_VMCS16_HOST_DS_SEL,
525 VMX_VMCS16_HOST_FS_SEL,
526 VMX_VMCS16_HOST_GS_SEL,
527 VMX_VMCS16_HOST_TR_SEL,
528
529 /* 64-bit control fields. */
530 VMX_VMCS64_CTRL_IO_BITMAP_A_FULL,
531 VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH,
532 VMX_VMCS64_CTRL_IO_BITMAP_B_FULL,
533 VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH,
534 VMX_VMCS64_CTRL_MSR_BITMAP_FULL,
535 VMX_VMCS64_CTRL_MSR_BITMAP_HIGH,
536 VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL,
537 VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH,
538 VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL,
539 VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH,
540 VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL,
541 VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH,
542 VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL,
543 VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH,
544 VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL,
545 VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH,
546 VMX_VMCS64_CTRL_TSC_OFFSET_FULL,
547 VMX_VMCS64_CTRL_TSC_OFFSET_HIGH,
548 VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL,
549 VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH,
550 VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL,
551 VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH,
552 VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL,
553 VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH,
554 VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL,
555 VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH,
556 VMX_VMCS64_CTRL_EPTP_FULL,
557 VMX_VMCS64_CTRL_EPTP_HIGH,
558 VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL,
559 VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH,
560 VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL,
561 VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH,
562 VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL,
563 VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH,
564 VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL,
565 VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH,
566 VMX_VMCS64_CTRL_EPTP_LIST_FULL,
567 VMX_VMCS64_CTRL_EPTP_LIST_HIGH,
568 VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL,
569 VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH,
570 VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL,
571 VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH,
572 VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_FULL,
573 VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_HIGH,
574 VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL,
575 VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH,
576 VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_FULL,
577 VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_HIGH,
578 VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL,
579 VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH,
580
581 /* 64-bit read-only data fields. */
582 VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,
583 VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH,
584
585 /* 64-bit guest-state fields. */
586 VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL,
587 VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH,
588 VMX_VMCS64_GUEST_DEBUGCTL_FULL,
589 VMX_VMCS64_GUEST_DEBUGCTL_HIGH,
590 VMX_VMCS64_GUEST_PAT_FULL,
591 VMX_VMCS64_GUEST_PAT_HIGH,
592 VMX_VMCS64_GUEST_EFER_FULL,
593 VMX_VMCS64_GUEST_EFER_HIGH,
594 VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL,
595 VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_HIGH,
596 VMX_VMCS64_GUEST_PDPTE0_FULL,
597 VMX_VMCS64_GUEST_PDPTE0_HIGH,
598 VMX_VMCS64_GUEST_PDPTE1_FULL,
599 VMX_VMCS64_GUEST_PDPTE1_HIGH,
600 VMX_VMCS64_GUEST_PDPTE2_FULL,
601 VMX_VMCS64_GUEST_PDPTE2_HIGH,
602 VMX_VMCS64_GUEST_PDPTE3_FULL,
603 VMX_VMCS64_GUEST_PDPTE3_HIGH,
604 VMX_VMCS64_GUEST_BNDCFGS_FULL,
605 VMX_VMCS64_GUEST_BNDCFGS_HIGH,
606
607 /* 64-bit host-state fields. */
608 VMX_VMCS64_HOST_PAT_FULL,
609 VMX_VMCS64_HOST_PAT_HIGH,
610 VMX_VMCS64_HOST_EFER_FULL,
611 VMX_VMCS64_HOST_EFER_HIGH,
612 VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL,
613 VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_HIGH,
614
615 /* 32-bit control fields. */
616 VMX_VMCS32_CTRL_PIN_EXEC,
617 VMX_VMCS32_CTRL_PROC_EXEC,
618 VMX_VMCS32_CTRL_EXCEPTION_BITMAP,
619 VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK,
620 VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH,
621 VMX_VMCS32_CTRL_CR3_TARGET_COUNT,
622 VMX_VMCS32_CTRL_EXIT,
623 VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT,
624 VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT,
625 VMX_VMCS32_CTRL_ENTRY,
626 VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT,
627 VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO,
628 VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE,
629 VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH,
630 VMX_VMCS32_CTRL_TPR_THRESHOLD,
631 VMX_VMCS32_CTRL_PROC_EXEC2,
632 VMX_VMCS32_CTRL_PLE_GAP,
633 VMX_VMCS32_CTRL_PLE_WINDOW,
634
635 /* 32-bits read-only fields. */
636 VMX_VMCS32_RO_VM_INSTR_ERROR,
637 VMX_VMCS32_RO_EXIT_REASON,
638 VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,
639 VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE,
640 VMX_VMCS32_RO_IDT_VECTORING_INFO,
641 VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,
642 VMX_VMCS32_RO_EXIT_INSTR_LENGTH,
643 VMX_VMCS32_RO_EXIT_INSTR_INFO,
644
645 /* 32-bit guest-state fields. */
646 VMX_VMCS32_GUEST_ES_LIMIT,
647 VMX_VMCS32_GUEST_CS_LIMIT,
648 VMX_VMCS32_GUEST_SS_LIMIT,
649 VMX_VMCS32_GUEST_DS_LIMIT,
650 VMX_VMCS32_GUEST_FS_LIMIT,
651 VMX_VMCS32_GUEST_GS_LIMIT,
652 VMX_VMCS32_GUEST_LDTR_LIMIT,
653 VMX_VMCS32_GUEST_TR_LIMIT,
654 VMX_VMCS32_GUEST_GDTR_LIMIT,
655 VMX_VMCS32_GUEST_IDTR_LIMIT,
656 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
657 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
658 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
659 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
660 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
661 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS,
662 VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS,
663 VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS,
664 VMX_VMCS32_GUEST_INT_STATE,
665 VMX_VMCS32_GUEST_ACTIVITY_STATE,
666 VMX_VMCS32_GUEST_SMBASE,
667 VMX_VMCS32_GUEST_SYSENTER_CS,
668 VMX_VMCS32_PREEMPT_TIMER_VALUE,
669
670 /* 32-bit host-state fields. */
671 VMX_VMCS32_HOST_SYSENTER_CS,
672
673 /* Natural-width control fields. */
674 VMX_VMCS_CTRL_CR0_MASK,
675 VMX_VMCS_CTRL_CR4_MASK,
676 VMX_VMCS_CTRL_CR0_READ_SHADOW,
677 VMX_VMCS_CTRL_CR4_READ_SHADOW,
678 VMX_VMCS_CTRL_CR3_TARGET_VAL0,
679 VMX_VMCS_CTRL_CR3_TARGET_VAL1,
680 VMX_VMCS_CTRL_CR3_TARGET_VAL2,
681 VMX_VMCS_CTRL_CR3_TARGET_VAL3,
682
683 /* Natural-width read-only data fields. */
684 VMX_VMCS_RO_EXIT_QUALIFICATION,
685 VMX_VMCS_RO_IO_RCX,
686 VMX_VMCS_RO_IO_RSI,
687 VMX_VMCS_RO_IO_RDI,
688 VMX_VMCS_RO_IO_RIP,
689 VMX_VMCS_RO_GUEST_LINEAR_ADDR,
690
691 /* Natural-width guest-state field */
692 VMX_VMCS_GUEST_CR0,
693 VMX_VMCS_GUEST_CR3,
694 VMX_VMCS_GUEST_CR4,
695 VMX_VMCS_GUEST_ES_BASE,
696 VMX_VMCS_GUEST_CS_BASE,
697 VMX_VMCS_GUEST_SS_BASE,
698 VMX_VMCS_GUEST_DS_BASE,
699 VMX_VMCS_GUEST_FS_BASE,
700 VMX_VMCS_GUEST_GS_BASE,
701 VMX_VMCS_GUEST_LDTR_BASE,
702 VMX_VMCS_GUEST_TR_BASE,
703 VMX_VMCS_GUEST_GDTR_BASE,
704 VMX_VMCS_GUEST_IDTR_BASE,
705 VMX_VMCS_GUEST_DR7,
706 VMX_VMCS_GUEST_RSP,
707 VMX_VMCS_GUEST_RIP,
708 VMX_VMCS_GUEST_RFLAGS,
709 VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,
710 VMX_VMCS_GUEST_SYSENTER_ESP,
711 VMX_VMCS_GUEST_SYSENTER_EIP,
712
713 /* Natural-width host-state fields */
714 VMX_VMCS_HOST_CR0,
715 VMX_VMCS_HOST_CR3,
716 VMX_VMCS_HOST_CR4,
717 VMX_VMCS_HOST_FS_BASE,
718 VMX_VMCS_HOST_GS_BASE,
719 VMX_VMCS_HOST_TR_BASE,
720 VMX_VMCS_HOST_GDTR_BASE,
721 VMX_VMCS_HOST_IDTR_BASE,
722 VMX_VMCS_HOST_SYSENTER_ESP,
723 VMX_VMCS_HOST_SYSENTER_EIP,
724 VMX_VMCS_HOST_RSP,
725 VMX_VMCS_HOST_RIP
726};
727#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
728
729#ifdef VMX_USE_CACHED_VMCS_ACCESSES
730static const uint32_t g_aVmcsCacheSegBase[] =
731{
732 VMX_VMCS_GUEST_ES_BASE_CACHE_IDX,
733 VMX_VMCS_GUEST_CS_BASE_CACHE_IDX,
734 VMX_VMCS_GUEST_SS_BASE_CACHE_IDX,
735 VMX_VMCS_GUEST_DS_BASE_CACHE_IDX,
736 VMX_VMCS_GUEST_FS_BASE_CACHE_IDX,
737 VMX_VMCS_GUEST_GS_BASE_CACHE_IDX
738};
739AssertCompile(RT_ELEMENTS(g_aVmcsCacheSegBase) == X86_SREG_COUNT);
740#endif
741static const uint32_t g_aVmcsSegBase[] =
742{
743 VMX_VMCS_GUEST_ES_BASE,
744 VMX_VMCS_GUEST_CS_BASE,
745 VMX_VMCS_GUEST_SS_BASE,
746 VMX_VMCS_GUEST_DS_BASE,
747 VMX_VMCS_GUEST_FS_BASE,
748 VMX_VMCS_GUEST_GS_BASE
749};
750static const uint32_t g_aVmcsSegSel[] =
751{
752 VMX_VMCS16_GUEST_ES_SEL,
753 VMX_VMCS16_GUEST_CS_SEL,
754 VMX_VMCS16_GUEST_SS_SEL,
755 VMX_VMCS16_GUEST_DS_SEL,
756 VMX_VMCS16_GUEST_FS_SEL,
757 VMX_VMCS16_GUEST_GS_SEL
758};
759static const uint32_t g_aVmcsSegLimit[] =
760{
761 VMX_VMCS32_GUEST_ES_LIMIT,
762 VMX_VMCS32_GUEST_CS_LIMIT,
763 VMX_VMCS32_GUEST_SS_LIMIT,
764 VMX_VMCS32_GUEST_DS_LIMIT,
765 VMX_VMCS32_GUEST_FS_LIMIT,
766 VMX_VMCS32_GUEST_GS_LIMIT
767};
768static const uint32_t g_aVmcsSegAttr[] =
769{
770 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
771 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
772 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
773 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
774 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
775 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
776};
777AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
778AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
779AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
780AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
781
782#ifdef HMVMX_USE_FUNCTION_TABLE
783/**
784 * VMX_EXIT dispatch table.
785 */
786static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
787{
788 /* 0 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
789 /* 1 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
790 /* 2 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
791 /* 3 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitErrUnexpected,
792 /* 4 VMX_EXIT_SIPI */ hmR0VmxExitErrUnexpected,
793 /* 5 VMX_EXIT_IO_SMI */ hmR0VmxExitErrUnexpected,
794 /* 6 VMX_EXIT_SMI */ hmR0VmxExitErrUnexpected,
795 /* 7 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
796 /* 8 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
797 /* 9 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
798 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
799 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
800 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
801 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
802 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
803 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
804 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
805 /* 17 VMX_EXIT_RSM */ hmR0VmxExitErrUnexpected,
806 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
807#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
808 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
809 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
810 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
811 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
812 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
813 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
814 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
815 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
816 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
817#else
818 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
819 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
820 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
821 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
822 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
823 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
824 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
825 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
826 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
827#endif
828 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
829 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
830 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
831 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
832 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
833 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
834 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrUnexpected,
835 /* 35 UNDEFINED */ hmR0VmxExitErrUnexpected,
836 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
837 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
838 /* 38 UNDEFINED */ hmR0VmxExitErrUnexpected,
839 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
840 /* 40 VMX_EXIT_PAUSE */ hmR0VmxExitPause,
841 /* 41 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUnexpected,
842 /* 42 UNDEFINED */ hmR0VmxExitErrUnexpected,
843 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
844 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
845 /* 45 VMX_EXIT_VIRTUALIZED_EOI */ hmR0VmxExitErrUnexpected,
846 /* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitErrUnexpected,
847 /* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitErrUnexpected,
848 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
849 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
850 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
851 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
852 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
853#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
854 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInvvpid,
855#else
856 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
857#endif
858 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
859 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
860 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUnexpected,
861 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitErrUnexpected,
862 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
863 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitErrUnexpected,
864 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUnexpected,
865 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUnexpected,
866 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUnexpected,
867 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitErrUnexpected,
868 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitErrUnexpected,
869 /* 65 UNDEFINED */ hmR0VmxExitErrUnexpected,
870 /* 66 VMX_EXIT_SPP_EVENT */ hmR0VmxExitErrUnexpected,
871 /* 67 VMX_EXIT_UMWAIT */ hmR0VmxExitErrUnexpected,
872 /* 68 VMX_EXIT_TPAUSE */ hmR0VmxExitErrUnexpected,
873};
874#endif /* HMVMX_USE_FUNCTION_TABLE */
875
876#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
877static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
878{
879 /* 0 */ "(Not Used)",
880 /* 1 */ "VMCALL executed in VMX root operation.",
881 /* 2 */ "VMCLEAR with invalid physical address.",
882 /* 3 */ "VMCLEAR with VMXON pointer.",
883 /* 4 */ "VMLAUNCH with non-clear VMCS.",
884 /* 5 */ "VMRESUME with non-launched VMCS.",
885 /* 6 */ "VMRESUME after VMXOFF",
886 /* 7 */ "VM-entry with invalid control fields.",
887 /* 8 */ "VM-entry with invalid host state fields.",
888 /* 9 */ "VMPTRLD with invalid physical address.",
889 /* 10 */ "VMPTRLD with VMXON pointer.",
890 /* 11 */ "VMPTRLD with incorrect revision identifier.",
891 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
892 /* 13 */ "VMWRITE to read-only VMCS component.",
893 /* 14 */ "(Not Used)",
894 /* 15 */ "VMXON executed in VMX root operation.",
895 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
896 /* 17 */ "VM-entry with non-launched executing VMCS.",
897 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
898 /* 19 */ "VMCALL with non-clear VMCS.",
899 /* 20 */ "VMCALL with invalid VM-exit control fields.",
900 /* 21 */ "(Not Used)",
901 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
902 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
903 /* 24 */ "VMCALL with invalid SMM-monitor features.",
904 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
905 /* 26 */ "VM-entry with events blocked by MOV SS.",
906 /* 27 */ "(Not Used)",
907 /* 28 */ "Invalid operand to INVEPT/INVVPID."
908};
909#endif /* VBOX_STRICT */
910
911
912/**
913 * Get the CR0 guest/host mask that does not change through the lifetime of a VM.
914 *
915 * Any bit set in this mask is owned by the host/hypervisor and would cause a
916 * VM-exit when modified by the guest.
917 *
918 * @returns The static CR0 guest/host mask.
919 * @param pVCpu The cross context virtual CPU structure.
920 */
921DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr0Mask(PCVMCPU pVCpu)
922{
923 /*
924 * Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
925 * to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
926 */
927 /** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
928 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
929 * and @bugref{6944}. */
930 PVM pVM = pVCpu->CTX_SUFF(pVM);
931 return ( X86_CR0_PE
932 | X86_CR0_NE
933 | (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
934 | X86_CR0_PG
935 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
936 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
937 | X86_CR0_NW); /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
938}
939
940
941/**
942 * Gets the CR4 guest/host mask that does not change through the lifetime of a VM.
943 *
944 * Any bit set in this mask is owned by the host/hypervisor and would cause a
945 * VM-exit when modified by the guest.
946 *
947 * @returns The static CR4 guest/host mask.
948 * @param pVCpu The cross context virtual CPU structure.
949 */
950DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr4Mask(PCVMCPU pVCpu)
951{
952 /*
953 * We need to look at the host features here (for e.g. OSXSAVE, PCID) because
954 * these bits are reserved on hardware that does not support them. Since the
955 * CPU cannot refer to our virtual CPUID, we need to intercept CR4 changes to
956 * these bits and handle it depending on whether we expose them to the guest.
957 */
958 PVM pVM = pVCpu->CTX_SUFF(pVM);
959 bool const fXSaveRstor = pVM->cpum.ro.HostFeatures.fXSaveRstor;
960 bool const fPcid = pVM->cpum.ro.HostFeatures.fPcid;
961 return ( X86_CR4_VMXE
962 | X86_CR4_VME
963 | X86_CR4_PAE
964 | X86_CR4_PGE
965 | X86_CR4_PSE
966 | (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
967 | (fPcid ? X86_CR4_PCIDE : 0));
968}
969
970
971/**
972 * Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
973 * area.
974 *
975 * @returns @c true if it's different, @c false otherwise.
976 * @param pVmcsInfo The VMCS info. object.
977 */
978DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
979{
980 return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
981 && pVmcsInfo->pvGuestMsrStore);
982}
983
984
985/**
986 * Checks whether one of the given Pin-based VM-execution controls are set.
987 *
988 * @returns @c true if set, @c false otherwise.
989 * @param pVCpu The cross context virtual CPU structure.
990 * @param pVmxTransient The VMX-transient structure.
991 * @param uPinCtls The Pin-based VM-execution controls to check.
992 *
993 * @remarks This will not check merged controls when executing a nested-guest
994 * but the original control specified by the guest hypervisor.
995 */
996static bool hmR0VmxIsPinCtlsSet(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uPinCtls)
997{
998 if (!pVmxTransient->fIsNestedGuest)
999 {
1000 PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1001 return RT_BOOL(pVmcsInfo->u32PinCtls & uPinCtls);
1002 }
1003 return CPUMIsGuestVmxPinCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uPinCtls);
1004}
1005
1006
1007/**
1008 * Checks whether one of the given Processor-based VM-execution controls are set.
1009 *
1010 * @returns @c true if set, @c false otherwise.
1011 * @param pVCpu The cross context virtual CPU structure.
1012 * @param pVmxTransient The VMX-transient structure.
1013 * @param uProcCtls The Processor-based VM-execution controls to check.
1014 *
1015 * @remarks This will not check merged controls when executing a nested-guest
1016 * but the original control specified by the guest hypervisor.
1017 */
1018static bool hmR0VmxIsProcCtlsSet(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
1019{
1020 if (!pVmxTransient->fIsNestedGuest)
1021 {
1022 PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1023 return RT_BOOL(pVmcsInfo->u32ProcCtls & uProcCtls);
1024 }
1025 return CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uProcCtls);
1026}
1027
1028
1029/**
1030 * Checks whether one of the given Secondary Processor-based VM-execution controls
1031 * are set.
1032 *
1033 * @returns @c true if set, @c false otherwise.
1034 * @param pVCpu The cross context virtual CPU structure.
1035 * @param pVmxTransient The VMX-transient structure.
1036 * @param uProcCtls2 The Secondary Processor-based VM-execution controls to
1037 * check.
1038 *
1039 * @remarks This will not check merged controls when executing a nested-guest
1040 * but the original control specified by the guest hypervisor.
1041 */
1042static bool hmR0VmxIsProcCtls2Set(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uProcCtls2)
1043{
1044 if (!pVmxTransient->fIsNestedGuest)
1045 {
1046 PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1047 return RT_BOOL(pVmcsInfo->u32ProcCtls2 & uProcCtls2);
1048 }
1049 return CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, uProcCtls2);
1050}
1051
1052
1053#if 0
1054/**
1055 * Checks whether one of the given VM-entry controls are set.
1056 *
1057 * @returns @c true if set, @c false otherwise.
1058 * @param pVCpu The cross context virtual CPU structure.
1059 * @param pVmxTransient The VMX-transient structure.
1060 * @param uEntryCtls The VM-entry controls to check.
1061 *
1062 * @remarks This will not check merged controls when executing a nested-guest
1063 * but the original control specified by the guest hypervisor.
1064 */
1065static bool hmR0VmxIsEntryCtlsSet(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uEntryCtls)
1066{
1067 if (!pVmxTransient->fIsNestedGuest)
1068 {
1069 PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1070 return RT_BOOL(pVmcsInfo->u32EntryCtls & uEntryCtls);
1071 }
1072 return CPUMIsGuestVmxEntryCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uEntryCtls);
1073}
1074
1075
1076/**
1077 * Checks whether one of the given VM-exit controls are set.
1078 *
1079 * @returns @c true if set, @c false otherwise.
1080 * @param pVCpu The cross context virtual CPU structure.
1081 * @param pVmxTransient The VMX-transient structure.
1082 * @param uExitCtls The VM-exit controls to check.
1083 *
1084 * @remarks This will not check merged controls when executing a nested-guest
1085 * but the original control specified by the guest hypervisor.
1086 */
1087static bool hmR0VmxIsExitCtlsSet(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uExitCtls)
1088{
1089 if (!pVmxTransient->fIsNestedGuest)
1090 {
1091 PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1092 return RT_BOOL(pVmcsInfo->u32ExitCtls & uExitCtls);
1093 }
1094 return CPUMIsGuestVmxExitCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uExitCtls);
1095}
1096#endif
1097
1098
1099/**
1100 * Adds one or more exceptions to the exception bitmap and commits it to the current
1101 * VMCS.
1102 *
1103 * @returns VBox status code.
1104 * @param pVmxTransient The VMX-transient structure.
1105 * @param uXcptMask The exception(s) to add.
1106 */
1107static int hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1108{
1109 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1110 uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
1111 if ((uXcptBitmap & uXcptMask) != uXcptMask)
1112 {
1113 uXcptBitmap |= uXcptMask;
1114 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
1115 AssertRCReturn(rc, rc);
1116 pVmcsInfo->u32XcptBitmap = uXcptBitmap;
1117 }
1118 return VINF_SUCCESS;
1119}
1120
1121
1122/**
1123 * Adds an exception to the exception bitmap and commits it to the current VMCS.
1124 *
1125 * @returns VBox status code.
1126 * @param pVmxTransient The VMX-transient structure.
1127 * @param uXcpt The exception to add.
1128 */
1129static int hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1130{
1131 Assert(uXcpt <= X86_XCPT_LAST);
1132 return hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
1133}
1134
1135
1136/**
1137 * Remove one or more exceptions from the exception bitmap and commits it to the
1138 * current VMCS.
1139 *
1140 * This takes care of not removing the exception intercept if a nested-guest
1141 * requires the exception to be intercepted.
1142 *
1143 * @returns VBox status code.
1144 * @param pVCpu The cross context virtual CPU structure.
1145 * @param pVmxTransient The VMX-transient structure.
1146 * @param uXcptMask The exception(s) to remove.
1147 */
1148static int hmR0VmxRemoveXcptInterceptMask(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1149{
1150 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1151 uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
1152 if (u32XcptBitmap & uXcptMask)
1153 {
1154#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1155 if (!pVmxTransient->fIsNestedGuest)
1156 { /* likely */ }
1157 else
1158 {
1159 PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
1160 uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
1161 }
1162#endif
1163#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
1164 uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
1165 | RT_BIT(X86_XCPT_DE)
1166 | RT_BIT(X86_XCPT_NM)
1167 | RT_BIT(X86_XCPT_TS)
1168 | RT_BIT(X86_XCPT_UD)
1169 | RT_BIT(X86_XCPT_NP)
1170 | RT_BIT(X86_XCPT_SS)
1171 | RT_BIT(X86_XCPT_GP)
1172 | RT_BIT(X86_XCPT_PF)
1173 | RT_BIT(X86_XCPT_MF));
1174#elif defined(HMVMX_ALWAYS_TRAP_PF)
1175 uXcptMask &= ~RT_BIT(X86_XCPT_PF);
1176#endif
1177 if (uXcptMask)
1178 {
1179 /* Validate we are not removing any essential exception intercepts. */
1180 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging || !(uXcptMask & RT_BIT(X86_XCPT_PF))); RT_NOREF(pVCpu);
1181 Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
1182 Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
1183
1184 /* Remove it from the exception bitmap. */
1185 u32XcptBitmap &= ~uXcptMask;
1186
1187 /* Commit and update the cache if necessary. */
1188 if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
1189 {
1190 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1191 AssertRCReturn(rc, rc);
1192 pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
1193 }
1194 }
1195 }
1196 return VINF_SUCCESS;
1197}
1198
1199
1200/**
1201 * Remove an exceptions from the exception bitmap and commits it to the current
1202 * VMCS.
1203 *
1204 * @returns VBox status code.
1205 * @param pVCpu The cross context virtual CPU structure.
1206 * @param pVmxTransient The VMX-transient structure.
1207 * @param uXcpt The exception to remove.
1208 */
1209static int hmR0VmxRemoveXcptIntercept(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1210{
1211 return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
1212}
1213
1214
1215/**
1216 * Loads the VMCS specified by the VMCS info. object.
1217 *
1218 * @returns VBox status code.
1219 * @param pVmcsInfo The VMCS info. object.
1220 *
1221 * @remarks Can be called with interrupts disabled.
1222 */
1223static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
1224{
1225 Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1226 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1227
1228 int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
1229 if (RT_SUCCESS(rc))
1230 pVmcsInfo->fVmcsState |= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1231 return rc;
1232}
1233
1234
1235/**
1236 * Clears the VMCS specified by the VMCS info. object.
1237 *
1238 * @returns VBox status code.
1239 * @param pVmcsInfo The VMCS info. object.
1240 *
1241 * @remarks Can be called with interrupts disabled.
1242 */
1243static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
1244{
1245 Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1246 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1247
1248 int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
1249 if (RT_SUCCESS(rc))
1250 pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1251 return rc;
1252}
1253
1254
1255#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1256/**
1257 * Loads the shadow VMCS specified by the VMCS info. object.
1258 *
1259 * @returns VBox status code.
1260 * @param pVmcsInfo The VMCS info. object.
1261 *
1262 * @remarks Can be called with interrupts disabled.
1263 */
1264static int hmR0VmxLoadShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1265{
1266 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1267 Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1268
1269 int rc = VMXLoadVmcs(pVmcsInfo->HCPhysShadowVmcs);
1270 if (RT_SUCCESS(rc))
1271 pVmcsInfo->fShadowVmcsState |= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1272 return rc;
1273}
1274
1275
1276/**
1277 * Clears the shadow VMCS specified by the VMCS info. object.
1278 *
1279 * @returns VBox status code.
1280 * @param pVmcsInfo The VMCS info. object.
1281 *
1282 * @remarks Can be called with interrupts disabled.
1283 */
1284static int hmR0VmxClearShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1285{
1286 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1287 Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1288
1289 int rc = VMXClearVmcs(pVmcsInfo->HCPhysShadowVmcs);
1290 if (RT_SUCCESS(rc))
1291 pVmcsInfo->fShadowVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1292 return rc;
1293}
1294
1295
1296/**
1297 * Switches the current VMCS to the one specified.
1298 *
1299 * @returns VBox status code.
1300 * @param pVmcsInfoFrom The VMCS info. object we are switching from.
1301 * @param pVmcsInfoTo The VMCS info. object we are switching to.
1302 *
1303 * @remarks Called with interrupts disabled.
1304 */
1305static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
1306{
1307 /*
1308 * Clear the VMCS we are switching out if it has not already been cleared.
1309 * This will sync any CPU internal data back to the VMCS.
1310 */
1311 if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1312 {
1313 int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
1314 if (RT_SUCCESS(rc))
1315 {
1316 /*
1317 * The shadow VMCS, if any, would not be active at this point since we
1318 * would have cleared it while importing the virtual hardware-virtualization
1319 * state as part the VMLAUNCH/VMRESUME VM-exit. Hence, there's no need to
1320 * clear the shadow VMCS here, just assert for safety.
1321 */
1322 Assert(!pVmcsInfoFrom->pvShadowVmcs || pVmcsInfoFrom->fShadowVmcsState == VMX_V_VMCS_LAUNCH_STATE_CLEAR);
1323 }
1324 else
1325 return rc;
1326 }
1327
1328 /*
1329 * Clear the VMCS we are switching to if it has not already been cleared.
1330 * This will initialize the VMCS launch state to "clear" required for loading it.
1331 *
1332 * See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
1333 */
1334 if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1335 {
1336 int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
1337 if (RT_SUCCESS(rc))
1338 { /* likely */ }
1339 else
1340 return rc;
1341 }
1342
1343 /*
1344 * Finally, load the VMCS we are switching to.
1345 */
1346 return hmR0VmxLoadVmcs(pVmcsInfoTo);
1347}
1348#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
1349
1350
1351/**
1352 * Updates the VM's last error record.
1353 *
1354 * If there was a VMX instruction error, reads the error data from the VMCS and
1355 * updates VCPU's last error record as well.
1356 *
1357 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
1358 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
1359 * VERR_VMX_INVALID_VMCS_FIELD.
1360 * @param rc The error code.
1361 */
1362static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
1363{
1364 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
1365 || rc == VERR_VMX_UNABLE_TO_START_VM)
1366 {
1367 AssertPtrReturnVoid(pVCpu);
1368 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
1369 }
1370 pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
1371}
1372
1373
1374#ifdef VBOX_STRICT
1375/**
1376 * Reads the VM-entry interruption-information field from the VMCS into the VMX
1377 * transient structure.
1378 *
1379 * @returns VBox status code.
1380 * @param pVmxTransient The VMX-transient structure.
1381 *
1382 * @remarks No-long-jump zone!!!
1383 */
1384DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1385{
1386 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
1387 AssertRCReturn(rc, rc);
1388 return VINF_SUCCESS;
1389}
1390
1391
1392/**
1393 * Reads the VM-entry exception error code field from the VMCS into
1394 * the VMX transient structure.
1395 *
1396 * @returns VBox status code.
1397 * @param pVmxTransient The VMX-transient structure.
1398 *
1399 * @remarks No-long-jump zone!!!
1400 */
1401DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1402{
1403 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
1404 AssertRCReturn(rc, rc);
1405 return VINF_SUCCESS;
1406}
1407
1408
1409/**
1410 * Reads the VM-entry exception error code field from the VMCS into
1411 * the VMX transient structure.
1412 *
1413 * @returns VBox status code.
1414 * @param pVmxTransient The VMX-transient structure.
1415 *
1416 * @remarks No-long-jump zone!!!
1417 */
1418DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1419{
1420 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
1421 AssertRCReturn(rc, rc);
1422 return VINF_SUCCESS;
1423}
1424#endif /* VBOX_STRICT */
1425
1426
1427/**
1428 * Reads the VM-exit interruption-information field from the VMCS into the VMX
1429 * transient structure.
1430 *
1431 * @returns VBox status code.
1432 * @param pVmxTransient The VMX-transient structure.
1433 */
1434DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1435{
1436 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
1437 {
1438 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1439 AssertRCReturn(rc,rc);
1440 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_INFO;
1441 }
1442 return VINF_SUCCESS;
1443}
1444
1445
1446/**
1447 * Reads the VM-exit interruption error code from the VMCS into the VMX
1448 * transient structure.
1449 *
1450 * @returns VBox status code.
1451 * @param pVmxTransient The VMX-transient structure.
1452 */
1453DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1454{
1455 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1456 {
1457 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1458 AssertRCReturn(rc, rc);
1459 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1460 }
1461 return VINF_SUCCESS;
1462}
1463
1464
1465/**
1466 * Reads the VM-exit instruction length field from the VMCS into the VMX
1467 * transient structure.
1468 *
1469 * @returns VBox status code.
1470 * @param pVmxTransient The VMX-transient structure.
1471 */
1472DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1473{
1474 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1475 {
1476 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1477 AssertRCReturn(rc, rc);
1478 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_LEN;
1479 }
1480 return VINF_SUCCESS;
1481}
1482
1483
1484/**
1485 * Reads the VM-exit instruction-information field from the VMCS into
1486 * the VMX transient structure.
1487 *
1488 * @returns VBox status code.
1489 * @param pVmxTransient The VMX-transient structure.
1490 */
1491DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1492{
1493 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1494 {
1495 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1496 AssertRCReturn(rc, rc);
1497 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_INFO;
1498 }
1499 return VINF_SUCCESS;
1500}
1501
1502
1503/**
1504 * Reads the Exit Qualification from the VMCS into the VMX transient structure.
1505 *
1506 * @returns VBox status code.
1507 * @param pVCpu The cross context virtual CPU structure of the
1508 * calling EMT. (Required for the VMCS cache case.)
1509 * @param pVmxTransient The VMX-transient structure.
1510 */
1511DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1512{
1513 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1514 {
1515 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
1516 AssertRCReturn(rc, rc);
1517 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION;
1518 }
1519 return VINF_SUCCESS;
1520}
1521
1522
1523/**
1524 * Reads the Guest-linear address from the VMCS into the VMX transient structure.
1525 *
1526 * @returns VBox status code.
1527 * @param pVCpu The cross context virtual CPU structure of the
1528 * calling EMT. (Required for the VMCS cache case.)
1529 * @param pVmxTransient The VMX-transient structure.
1530 */
1531DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1532{
1533 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1534 {
1535 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
1536 AssertRCReturn(rc, rc);
1537 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_GUEST_LINEAR_ADDR;
1538 }
1539 return VINF_SUCCESS;
1540}
1541
1542
1543/**
1544 * Reads the IDT-vectoring information field from the VMCS into the VMX
1545 * transient structure.
1546 *
1547 * @returns VBox status code.
1548 * @param pVmxTransient The VMX-transient structure.
1549 *
1550 * @remarks No-long-jump zone!!!
1551 */
1552DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1553{
1554 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1555 {
1556 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1557 AssertRCReturn(rc, rc);
1558 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_INFO;
1559 }
1560 return VINF_SUCCESS;
1561}
1562
1563
1564/**
1565 * Reads the IDT-vectoring error code from the VMCS into the VMX
1566 * transient structure.
1567 *
1568 * @returns VBox status code.
1569 * @param pVmxTransient The VMX-transient structure.
1570 */
1571DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1572{
1573 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1574 {
1575 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1576 AssertRCReturn(rc, rc);
1577 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1578 }
1579 return VINF_SUCCESS;
1580}
1581
1582
1583/**
1584 * Enters VMX root mode operation on the current CPU.
1585 *
1586 * @returns VBox status code.
1587 * @param pVM The cross context VM structure. Can be
1588 * NULL, after a resume.
1589 * @param HCPhysCpuPage Physical address of the VMXON region.
1590 * @param pvCpuPage Pointer to the VMXON region.
1591 */
1592static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1593{
1594 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1595 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1596 Assert(pvCpuPage);
1597 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1598
1599 if (pVM)
1600 {
1601 /* Write the VMCS revision identifier to the VMXON region. */
1602 *(uint32_t *)pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1603 }
1604
1605 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1606 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1607
1608 /* Enable the VMX bit in CR4 if necessary. */
1609 RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1610
1611 /* Enter VMX root mode. */
1612 int rc = VMXEnable(HCPhysCpuPage);
1613 if (RT_FAILURE(rc))
1614 {
1615 if (!(uOldCr4 & X86_CR4_VMXE))
1616 SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1617
1618 if (pVM)
1619 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1620 }
1621
1622 /* Restore interrupts. */
1623 ASMSetFlags(fEFlags);
1624 return rc;
1625}
1626
1627
1628/**
1629 * Exits VMX root mode operation on the current CPU.
1630 *
1631 * @returns VBox status code.
1632 */
1633static int hmR0VmxLeaveRootMode(void)
1634{
1635 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1636
1637 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1638 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1639
1640 /* If we're for some reason not in VMX root mode, then don't leave it. */
1641 RTCCUINTREG const uHostCr4 = ASMGetCR4();
1642
1643 int rc;
1644 if (uHostCr4 & X86_CR4_VMXE)
1645 {
1646 /* Exit VMX root mode and clear the VMX bit in CR4. */
1647 VMXDisable();
1648 SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1649 rc = VINF_SUCCESS;
1650 }
1651 else
1652 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1653
1654 /* Restore interrupts. */
1655 ASMSetFlags(fEFlags);
1656 return rc;
1657}
1658
1659
1660/**
1661 * Allocates and maps a physically contiguous page. The allocated page is
1662 * zero'd out (used by various VT-x structures).
1663 *
1664 * @returns IPRT status code.
1665 * @param pMemObj Pointer to the ring-0 memory object.
1666 * @param ppVirt Where to store the virtual address of the
1667 * allocation.
1668 * @param pHCPhys Where to store the physical address of the
1669 * allocation.
1670 */
1671static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1672{
1673 AssertPtr(pMemObj);
1674 AssertPtr(ppVirt);
1675 AssertPtr(pHCPhys);
1676 int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1677 if (RT_FAILURE(rc))
1678 return rc;
1679 *ppVirt = RTR0MemObjAddress(*pMemObj);
1680 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
1681 ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1682 return VINF_SUCCESS;
1683}
1684
1685
1686/**
1687 * Frees and unmaps an allocated, physical page.
1688 *
1689 * @param pMemObj Pointer to the ring-0 memory object.
1690 * @param ppVirt Where to re-initialize the virtual address of
1691 * allocation as 0.
1692 * @param pHCPhys Where to re-initialize the physical address of the
1693 * allocation as 0.
1694 */
1695static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1696{
1697 AssertPtr(pMemObj);
1698 AssertPtr(ppVirt);
1699 AssertPtr(pHCPhys);
1700 /* NULL is valid, accepted and ignored by the free function below. */
1701 RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
1702 *pMemObj = NIL_RTR0MEMOBJ;
1703 *ppVirt = NULL;
1704 *pHCPhys = NIL_RTHCPHYS;
1705}
1706
1707
1708/**
1709 * Initializes a VMCS info. object.
1710 *
1711 * @param pVmcsInfo The VMCS info. object.
1712 */
1713static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1714{
1715 memset(pVmcsInfo, 0, sizeof(*pVmcsInfo));
1716
1717 Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1718 Assert(pVmcsInfo->hMemObjShadowVmcs == NIL_RTR0MEMOBJ);
1719 Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1720 Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1721 Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1722 Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1723 pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1724 pVmcsInfo->HCPhysShadowVmcs = NIL_RTHCPHYS;
1725 pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1726 pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1727 pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1728 pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1729 pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1730 pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1731 pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1732}
1733
1734
1735/**
1736 * Frees the VT-x structures for a VMCS info. object.
1737 *
1738 * @param pVM The cross context VM structure.
1739 * @param pVmcsInfo The VMCS info. object.
1740 */
1741static void hmR0VmxFreeVmcsInfo(PVM pVM, PVMXVMCSINFO pVmcsInfo)
1742{
1743 hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1744
1745#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1746 if (pVM->hm.s.vmx.fUseVmcsShadowing)
1747 hmR0VmxPageFree(&pVmcsInfo->hMemObjShadowVmcs, &pVmcsInfo->pvShadowVmcs, &pVmcsInfo->HCPhysShadowVmcs);
1748#endif
1749
1750 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1751 hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1752
1753 hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1754 hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1755 hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1756
1757 hmR0VmxInitVmcsInfo(pVmcsInfo);
1758}
1759
1760
1761/**
1762 * Allocates the VT-x structures for a VMCS info. object.
1763 *
1764 * @returns VBox status code.
1765 * @param pVCpu The cross context virtual CPU structure.
1766 * @param pVmcsInfo The VMCS info. object.
1767 * @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1768 */
1769static int hmR0VmxAllocVmcsInfo(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1770{
1771 PVM pVM = pVCpu->CTX_SUFF(pVM);
1772
1773 /* Allocate the guest VM control structure (VMCS). */
1774 int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1775 if (RT_SUCCESS(rc))
1776 {
1777 if (!fIsNstGstVmcs)
1778 {
1779#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1780 if (pVM->hm.s.vmx.fUseVmcsShadowing)
1781 rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjShadowVmcs, &pVmcsInfo->pvShadowVmcs, &pVmcsInfo->HCPhysShadowVmcs);
1782#endif
1783 if (RT_SUCCESS(rc))
1784 {
1785 /* Get the allocated virtual-APIC page from the virtual APIC device. */
1786 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1787 && (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1788 {
1789 rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic,
1790 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1791 }
1792 }
1793 }
1794 else
1795 {
1796 /* We don't yet support exposing VMCS shadowing to the guest. */
1797 Assert(pVmcsInfo->HCPhysShadowVmcs == NIL_RTHCPHYS);
1798 Assert(!pVmcsInfo->pvShadowVmcs);
1799
1800 /* The host-physical address of the virtual-APIC page in guest memory is taken directly. */
1801 Assert(pVmcsInfo->HCPhysVirtApic == NIL_RTHCPHYS);
1802 Assert(!pVmcsInfo->pbVirtApic);
1803 }
1804
1805 if (RT_SUCCESS(rc))
1806 {
1807 /*
1808 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1809 * transparent accesses of specific MSRs.
1810 *
1811 * If the condition for enabling MSR bitmaps changes here, don't forget to
1812 * update HMIsMsrBitmapActive().
1813 *
1814 * We don't share MSR bitmaps between the guest and nested-guest as we then
1815 * don't need to care about carefully restoring the guest MSR bitmap.
1816 * The guest visible nested-guest MSR bitmap needs to remain unchanged.
1817 * Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1818 * We also don't need to re-initialize the nested-guest MSR bitmap here as
1819 * we do that later while merging VMCS.
1820 */
1821 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1822 {
1823 rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1824 if ( RT_SUCCESS(rc)
1825 && !fIsNstGstVmcs)
1826 ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1827 }
1828
1829 if (RT_SUCCESS(rc))
1830 {
1831 /*
1832 * Allocate the VM-entry MSR-load area for the guest MSRs.
1833 *
1834 * Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1835 * the guest and nested-guest.
1836 */
1837 rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1838 &pVmcsInfo->HCPhysGuestMsrLoad);
1839 if (RT_SUCCESS(rc))
1840 {
1841 /*
1842 * We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1843 * These contain the guest MSRs to load on VM-entry and store on VM-exit.
1844 */
1845 Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1846 pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1847 pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1848
1849 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1850 rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1851 &pVmcsInfo->HCPhysHostMsrLoad);
1852 }
1853 }
1854 }
1855 }
1856
1857 return rc;
1858}
1859
1860
1861/**
1862 * Free all VT-x structures for the VM.
1863 *
1864 * @returns IPRT status code.
1865 * @param pVM The cross context VM structure.
1866 */
1867static void hmR0VmxStructsFree(PVM pVM)
1868{
1869#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1870 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1871#endif
1872 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1873
1874#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1875 if (pVM->hm.s.vmx.fUseVmcsShadowing)
1876 {
1877 RTMemFree(pVM->hm.s.vmx.paShadowVmcsFields);
1878 RTMemFree(pVM->hm.s.vmx.paShadowVmcsRoFields);
1879 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap, &pVM->hm.s.vmx.HCPhysVmreadBitmap);
1880 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap, &pVM->hm.s.vmx.HCPhysVmwriteBitmap);
1881 }
1882#endif
1883
1884 for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1885 {
1886 PVMCPU pVCpu = &pVM->aCpus[idCpu];
1887 PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1888 hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1889#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1890 if (pVM->cpum.ro.GuestFeatures.fVmx)
1891 {
1892 pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1893 hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1894 }
1895#endif
1896 }
1897}
1898
1899
1900/**
1901 * Allocate all VT-x structures for the VM.
1902 *
1903 * @returns IPRT status code.
1904 * @param pVM The cross context VM structure.
1905 */
1906static int hmR0VmxStructsAlloc(PVM pVM)
1907{
1908 /*
1909 * Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1910 * The VMCS size cannot be more than 4096 bytes.
1911 *
1912 * See Intel spec. Appendix A.1 "Basic VMX Information".
1913 */
1914 uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1915 if (cbVmcs <= X86_PAGE_4K_SIZE)
1916 { /* likely */ }
1917 else
1918 {
1919 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1920 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1921 }
1922
1923 /*
1924 * Initialize/check members up-front so we can cleanup en masse on allocation failures.
1925 */
1926#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1927 Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1928 Assert(pVM->hm.s.vmx.pbScratch == NULL);
1929 pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1930#endif
1931
1932 Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1933 Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1934 pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1935
1936 for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1937 {
1938 PVMCPU pVCpu = &pVM->aCpus[idCpu];
1939 hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1940 hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1941 }
1942
1943 /*
1944 * Allocate per-VM VT-x structures.
1945 */
1946 int rc = VINF_SUCCESS;
1947#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1948 /* Allocate crash-dump magic scratch page. */
1949 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1950 if (RT_FAILURE(rc))
1951 {
1952 hmR0VmxStructsFree(pVM);
1953 return rc;
1954 }
1955 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1956 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1957#endif
1958
1959 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1960 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1961 {
1962 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1963 &pVM->hm.s.vmx.HCPhysApicAccess);
1964 if (RT_FAILURE(rc))
1965 {
1966 hmR0VmxStructsFree(pVM);
1967 return rc;
1968 }
1969 }
1970
1971#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1972 /* Allocate the shadow VMCS fields array, VMREAD, VMWRITE bitmaps.. */
1973 if (pVM->hm.s.vmx.fUseVmcsShadowing)
1974 {
1975 Assert(!pVM->hm.s.vmx.cShadowVmcsFields);
1976 Assert(!pVM->hm.s.vmx.cShadowVmcsRoFields);
1977 pVM->hm.s.vmx.paShadowVmcsFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
1978 pVM->hm.s.vmx.paShadowVmcsRoFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
1979 if (RT_LIKELY( pVM->hm.s.vmx.paShadowVmcsFields
1980 && pVM->hm.s.vmx.paShadowVmcsRoFields))
1981 {
1982 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap,
1983 &pVM->hm.s.vmx.HCPhysVmreadBitmap);
1984 if (RT_SUCCESS(rc))
1985 {
1986 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap,
1987 &pVM->hm.s.vmx.HCPhysVmwriteBitmap);
1988 }
1989 }
1990 else
1991 rc = VERR_NO_MEMORY;
1992
1993 if (RT_FAILURE(rc))
1994 {
1995 hmR0VmxStructsFree(pVM);
1996 return rc;
1997 }
1998 }
1999#endif
2000
2001 /*
2002 * Initialize per-VCPU VT-x structures.
2003 */
2004 for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
2005 {
2006 /* Allocate the guest VMCS structures. */
2007 PVMCPU pVCpu = &pVM->aCpus[idCpu];
2008 rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
2009 if (RT_SUCCESS(rc))
2010 {
2011#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2012 /* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
2013 if (pVM->cpum.ro.GuestFeatures.fVmx)
2014 {
2015 rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
2016 if (RT_SUCCESS(rc))
2017 { /* likely */ }
2018 else
2019 break;
2020 }
2021#endif
2022 }
2023 else
2024 break;
2025 }
2026
2027 if (RT_FAILURE(rc))
2028 {
2029 hmR0VmxStructsFree(pVM);
2030 return rc;
2031 }
2032
2033 return VINF_SUCCESS;
2034}
2035
2036#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2037/**
2038 * Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
2039 *
2040 * @returns @c true if the MSR is intercepted, @c false otherwise.
2041 * @param pvMsrBitmap The MSR bitmap.
2042 * @param offMsr The MSR byte offset.
2043 * @param iBit The bit offset from the byte offset.
2044 */
2045DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
2046{
2047 uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
2048 Assert(pbMsrBitmap);
2049 Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
2050 return ASMBitTest(pbMsrBitmap + offMsr, iBit);
2051}
2052#endif
2053
2054/**
2055 * Sets the permission bits for the specified MSR in the given MSR bitmap.
2056 *
2057 * If the passed VMCS is a nested-guest VMCS, this function ensures that the
2058 * read/write intercept is cleared from the MSR bitmap used for hardware-assisted
2059 * VMX execution of the nested-guest, only if nested-guest is also not intercepting
2060 * the read/write access of this MSR.
2061 *
2062 * @param pVCpu The cross context virtual CPU structure.
2063 * @param pVmcsInfo The VMCS info. object.
2064 * @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2065 * @param idMsr The MSR value.
2066 * @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
2067 * include both a read -and- a write permission!
2068 *
2069 * @sa CPUMGetVmxMsrPermission.
2070 * @remarks Can be called with interrupts disabled.
2071 */
2072static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
2073{
2074 uint8_t *pbMsrBitmap = (uint8_t *)pVmcsInfo->pvMsrBitmap;
2075 Assert(pbMsrBitmap);
2076 Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
2077
2078 /*
2079 * MSR-bitmap Layout:
2080 * Byte index MSR range Interpreted as
2081 * 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
2082 * 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
2083 * 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
2084 * 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
2085 *
2086 * A bit corresponding to an MSR within the above range causes a VM-exit
2087 * if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
2088 * the MSR range, it always cause a VM-exit.
2089 *
2090 * See Intel spec. 24.6.9 "MSR-Bitmap Address".
2091 */
2092 uint16_t const offBitmapRead = 0;
2093 uint16_t const offBitmapWrite = 0x800;
2094 uint16_t offMsr;
2095 int32_t iBit;
2096 if (idMsr <= UINT32_C(0x00001fff))
2097 {
2098 offMsr = 0;
2099 iBit = idMsr;
2100 }
2101 else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
2102 {
2103 offMsr = 0x400;
2104 iBit = idMsr - UINT32_C(0xc0000000);
2105 }
2106 else
2107 AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
2108
2109 /*
2110 * Set the MSR read permission.
2111 */
2112 uint16_t const offMsrRead = offBitmapRead + offMsr;
2113 Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
2114 if (fMsrpm & VMXMSRPM_ALLOW_RD)
2115 {
2116#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2117 bool const fClear = !fIsNstGstVmcs ? true
2118 : !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
2119#else
2120 RT_NOREF2(pVCpu, fIsNstGstVmcs);
2121 bool const fClear = true;
2122#endif
2123 if (fClear)
2124 ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
2125 }
2126 else
2127 ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
2128
2129 /*
2130 * Set the MSR write permission.
2131 */
2132 uint16_t const offMsrWrite = offBitmapWrite + offMsr;
2133 Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
2134 if (fMsrpm & VMXMSRPM_ALLOW_WR)
2135 {
2136#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2137 bool const fClear = !fIsNstGstVmcs ? true
2138 : !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
2139#else
2140 RT_NOREF2(pVCpu, fIsNstGstVmcs);
2141 bool const fClear = true;
2142#endif
2143 if (fClear)
2144 ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
2145 }
2146 else
2147 ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
2148}
2149
2150
2151/**
2152 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
2153 * area.
2154 *
2155 * @returns VBox status code.
2156 * @param pVCpu The cross context virtual CPU structure.
2157 * @param pVmcsInfo The VMCS info. object.
2158 * @param cMsrs The number of MSRs.
2159 */
2160static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
2161{
2162 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2163 uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
2164 if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
2165 {
2166 /* Commit the MSR counts to the VMCS and update the cache. */
2167 if (pVmcsInfo->cEntryMsrLoad != cMsrs)
2168 {
2169 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
2170 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
2171 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
2172 AssertRCReturn(rc, rc);
2173
2174 pVmcsInfo->cEntryMsrLoad = cMsrs;
2175 pVmcsInfo->cExitMsrStore = cMsrs;
2176 pVmcsInfo->cExitMsrLoad = cMsrs;
2177 }
2178 return VINF_SUCCESS;
2179 }
2180
2181 LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
2182 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
2183 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2184}
2185
2186
2187/**
2188 * Adds a new (or updates the value of an existing) guest/host MSR
2189 * pair to be swapped during the world-switch as part of the
2190 * auto-load/store MSR area in the VMCS.
2191 *
2192 * @returns VBox status code.
2193 * @param pVCpu The cross context virtual CPU structure.
2194 * @param pVmxTransient The VMX-transient structure.
2195 * @param idMsr The MSR.
2196 * @param uGuestMsrValue Value of the guest MSR.
2197 * @param fSetReadWrite Whether to set the guest read/write access of this
2198 * MSR (thus not causing a VM-exit).
2199 * @param fUpdateHostMsr Whether to update the value of the host MSR if
2200 * necessary.
2201 */
2202static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
2203 bool fSetReadWrite, bool fUpdateHostMsr)
2204{
2205 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2206 bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2207 PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2208 uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2209 uint32_t i;
2210
2211 /* Paranoia. */
2212 Assert(pGuestMsrLoad);
2213
2214 LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
2215
2216 /* Check if the MSR already exists in the VM-entry MSR-load area. */
2217 for (i = 0; i < cMsrs; i++)
2218 {
2219 if (pGuestMsrLoad[i].u32Msr == idMsr)
2220 break;
2221 }
2222
2223 bool fAdded = false;
2224 if (i == cMsrs)
2225 {
2226 /* The MSR does not exist, bump the MSR count to make room for the new MSR. */
2227 ++cMsrs;
2228 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2229 AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
2230
2231 /* Set the guest to read/write this MSR without causing VM-exits. */
2232 if ( fSetReadWrite
2233 && (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
2234 hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
2235
2236 LogFlowFunc(("MSR added, cMsrs now %u\n", cMsrs));
2237 fAdded = true;
2238 }
2239
2240 /* Update the MSR value for the newly added or already existing MSR. */
2241 pGuestMsrLoad[i].u32Msr = idMsr;
2242 pGuestMsrLoad[i].u64Value = uGuestMsrValue;
2243
2244 /* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
2245 if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2246 {
2247 PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2248 pGuestMsrStore[i].u32Msr = idMsr;
2249 pGuestMsrStore[i].u64Value = uGuestMsrValue;
2250 }
2251
2252 /* Update the corresponding slot in the host MSR area. */
2253 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2254 Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
2255 Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
2256 pHostMsr[i].u32Msr = idMsr;
2257
2258 /*
2259 * Only if the caller requests to update the host MSR value AND we've newly added the
2260 * MSR to the host MSR area do we actually update the value. Otherwise, it will be
2261 * updated by hmR0VmxUpdateAutoLoadHostMsrs().
2262 *
2263 * We do this for performance reasons since reading MSRs may be quite expensive.
2264 */
2265 if (fAdded)
2266 {
2267 if (fUpdateHostMsr)
2268 {
2269 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2270 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2271 pHostMsr[i].u64Value = ASMRdMsr(idMsr);
2272 }
2273 else
2274 {
2275 /* Someone else can do the work. */
2276 pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
2277 }
2278 }
2279 return VINF_SUCCESS;
2280}
2281
2282
2283/**
2284 * Removes a guest/host MSR pair to be swapped during the world-switch from the
2285 * auto-load/store MSR area in the VMCS.
2286 *
2287 * @returns VBox status code.
2288 * @param pVCpu The cross context virtual CPU structure.
2289 * @param pVmxTransient The VMX-transient structure.
2290 * @param idMsr The MSR.
2291 */
2292static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
2293{
2294 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2295 bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2296 PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2297 uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2298
2299 LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
2300
2301 for (uint32_t i = 0; i < cMsrs; i++)
2302 {
2303 /* Find the MSR. */
2304 if (pGuestMsrLoad[i].u32Msr == idMsr)
2305 {
2306 /*
2307 * If it's the last MSR, we only need to reduce the MSR count.
2308 * If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
2309 */
2310 if (i < cMsrs - 1)
2311 {
2312 /* Remove it from the VM-entry MSR-load area. */
2313 pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
2314 pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
2315
2316 /* Remove it from the VM-exit MSR-store area if it's in a different page. */
2317 if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2318 {
2319 PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2320 Assert(pGuestMsrStore[i].u32Msr == idMsr);
2321 pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
2322 pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
2323 }
2324
2325 /* Remove it from the VM-exit MSR-load area. */
2326 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2327 Assert(pHostMsr[i].u32Msr == idMsr);
2328 pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
2329 pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
2330 }
2331
2332 /* Reduce the count to reflect the removed MSR and bail. */
2333 --cMsrs;
2334 break;
2335 }
2336 }
2337
2338 /* Update the VMCS if the count changed (meaning the MSR was found and removed). */
2339 if (cMsrs != pVmcsInfo->cEntryMsrLoad)
2340 {
2341 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2342 AssertRCReturn(rc, rc);
2343
2344 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
2345 if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2346 hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD | VMXMSRPM_EXIT_WR);
2347
2348 Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
2349 return VINF_SUCCESS;
2350 }
2351
2352 return VERR_NOT_FOUND;
2353}
2354
2355
2356/**
2357 * Checks if the specified guest MSR is part of the VM-entry MSR-load area.
2358 *
2359 * @returns @c true if found, @c false otherwise.
2360 * @param pVmcsInfo The VMCS info. object.
2361 * @param idMsr The MSR to find.
2362 */
2363static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
2364{
2365 PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2366 uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
2367 Assert(pMsrs);
2368 Assert(sizeof(*pMsrs) * cMsrs <= X86_PAGE_4K_SIZE);
2369 for (uint32_t i = 0; i < cMsrs; i++)
2370 {
2371 if (pMsrs[i].u32Msr == idMsr)
2372 return true;
2373 }
2374 return false;
2375}
2376
2377
2378/**
2379 * Updates the value of all host MSRs in the VM-exit MSR-load area.
2380 *
2381 * @param pVCpu The cross context virtual CPU structure.
2382 * @param pVmcsInfo The VMCS info. object.
2383 *
2384 * @remarks No-long-jump zone!!!
2385 */
2386static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2387{
2388 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2389
2390 PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2391 uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
2392 Assert(pHostMsrLoad);
2393 Assert(sizeof(*pHostMsrLoad) * cMsrs <= X86_PAGE_4K_SIZE);
2394 LogFlowFunc(("pVCpu=%p cMsrs=%u\n", pVCpu, cMsrs));
2395 for (uint32_t i = 0; i < cMsrs; i++)
2396 {
2397 /*
2398 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
2399 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
2400 */
2401 if (pHostMsrLoad[i].u32Msr == MSR_K6_EFER)
2402 pHostMsrLoad[i].u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2403 else
2404 pHostMsrLoad[i].u64Value = ASMRdMsr(pHostMsrLoad[i].u32Msr);
2405 }
2406}
2407
2408
2409/**
2410 * Saves a set of host MSRs to allow read/write passthru access to the guest and
2411 * perform lazy restoration of the host MSRs while leaving VT-x.
2412 *
2413 * @param pVCpu The cross context virtual CPU structure.
2414 *
2415 * @remarks No-long-jump zone!!!
2416 */
2417static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
2418{
2419 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2420
2421 /*
2422 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
2423 */
2424 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
2425 {
2426 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
2427#if HC_ARCH_BITS == 64
2428 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2429 {
2430 pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
2431 pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
2432 pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
2433 pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
2434 }
2435#endif
2436 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
2437 }
2438}
2439
2440
2441/**
2442 * Checks whether the MSR belongs to the set of guest MSRs that we restore
2443 * lazily while leaving VT-x.
2444 *
2445 * @returns true if it does, false otherwise.
2446 * @param pVCpu The cross context virtual CPU structure.
2447 * @param idMsr The MSR to check.
2448 */
2449static bool hmR0VmxIsLazyGuestMsr(PCVMCPU pVCpu, uint32_t idMsr)
2450{
2451 NOREF(pVCpu);
2452#if HC_ARCH_BITS == 64
2453 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2454 {
2455 switch (idMsr)
2456 {
2457 case MSR_K8_LSTAR:
2458 case MSR_K6_STAR:
2459 case MSR_K8_SF_MASK:
2460 case MSR_K8_KERNEL_GS_BASE:
2461 return true;
2462 }
2463 }
2464#else
2465 RT_NOREF(pVCpu, idMsr);
2466#endif
2467 return false;
2468}
2469
2470
2471/**
2472 * Loads a set of guests MSRs to allow read/passthru to the guest.
2473 *
2474 * The name of this function is slightly confusing. This function does NOT
2475 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
2476 * common prefix for functions dealing with "lazy restoration" of the shared
2477 * MSRs.
2478 *
2479 * @param pVCpu The cross context virtual CPU structure.
2480 *
2481 * @remarks No-long-jump zone!!!
2482 */
2483static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
2484{
2485 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2486 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2487
2488 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2489#if HC_ARCH_BITS == 64
2490 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2491 {
2492 /*
2493 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
2494 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
2495 * we can skip a few MSR writes.
2496 *
2497 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
2498 * guest MSR values in the guest-CPU context might be different to what's currently
2499 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
2500 * CPU, see @bugref{8728}.
2501 */
2502 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2503 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2504 && pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
2505 && pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
2506 && pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
2507 && pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
2508 {
2509#ifdef VBOX_STRICT
2510 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
2511 Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
2512 Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
2513 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
2514#endif
2515 }
2516 else
2517 {
2518 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
2519 ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2520 ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2521 ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2522 }
2523 }
2524#endif
2525 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
2526}
2527
2528
2529/**
2530 * Performs lazy restoration of the set of host MSRs if they were previously
2531 * loaded with guest MSR values.
2532 *
2533 * @param pVCpu The cross context virtual CPU structure.
2534 *
2535 * @remarks No-long-jump zone!!!
2536 * @remarks The guest MSRs should have been saved back into the guest-CPU
2537 * context by hmR0VmxImportGuestState()!!!
2538 */
2539static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
2540{
2541 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2542 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2543
2544 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2545 {
2546 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2547#if HC_ARCH_BITS == 64
2548 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2549 {
2550 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2551 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2552 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2553 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2554 }
2555#endif
2556 }
2557 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
2558}
2559
2560
2561/**
2562 * Verifies that our cached values of the VMCS fields are all consistent with
2563 * what's actually present in the VMCS.
2564 *
2565 * @returns VBox status code.
2566 * @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2567 * @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2568 * VMCS content. HMCPU error-field is
2569 * updated, see VMX_VCI_XXX.
2570 * @param pVCpu The cross context virtual CPU structure.
2571 * @param pVmcsInfo The VMCS info. object.
2572 */
2573static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2574{
2575 uint32_t u32Val;
2576 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2577 AssertRCReturn(rc, rc);
2578 AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2579 ("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32EntryCtls, u32Val),
2580 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2581 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2582
2583 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2584 AssertRCReturn(rc, rc);
2585 AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2586 ("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ExitCtls, u32Val),
2587 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2588 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2589
2590 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2591 AssertRCReturn(rc, rc);
2592 AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2593 ("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32PinCtls, u32Val),
2594 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2595 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2596
2597 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2598 AssertRCReturn(rc, rc);
2599 AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2600 ("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls, u32Val),
2601 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2602 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2603
2604 if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2605 {
2606 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2607 AssertRCReturn(rc, rc);
2608 AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2609 ("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls2, u32Val),
2610 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2611 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2612 }
2613
2614 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2615 AssertRCReturn(rc, rc);
2616 AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2617 ("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32XcptBitmap, u32Val),
2618 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2619 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2620
2621 uint64_t u64Val;
2622 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2623 AssertRCReturn(rc, rc);
2624 AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2625 ("Cache=%#RX64 VMCS=%#RX64\n", pVmcsInfo->u64TscOffset, u64Val),
2626 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2627 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2628
2629 return VINF_SUCCESS;
2630}
2631
2632
2633#ifdef VBOX_STRICT
2634/**
2635 * Verifies that our cached host EFER MSR value has not changed since we cached it.
2636 *
2637 * @param pVCpu The cross context virtual CPU structure.
2638 * @param pVmcsInfo The VMCS info. object.
2639 */
2640static void hmR0VmxCheckHostEferMsr(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2641{
2642 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2643
2644 if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2645 {
2646 uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2647 uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2648 uint64_t uVmcsEferMsrVmcs;
2649 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2650 AssertRC(rc);
2651
2652 AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2653 ("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2654 AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2655 ("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2656 }
2657}
2658
2659
2660/**
2661 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2662 * VMCS are correct.
2663 *
2664 * @param pVCpu The cross context virtual CPU structure.
2665 * @param pVmcsInfo The VMCS info. object.
2666 * @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2667 */
2668static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2669{
2670 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2671
2672 /* Read the various MSR-area counts from the VMCS. */
2673 uint32_t cEntryLoadMsrs;
2674 uint32_t cExitStoreMsrs;
2675 uint32_t cExitLoadMsrs;
2676 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cEntryLoadMsrs); AssertRC(rc);
2677 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cExitStoreMsrs); AssertRC(rc);
2678 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cExitLoadMsrs); AssertRC(rc);
2679
2680 /* Verify all the MSR counts are the same. */
2681 Assert(cEntryLoadMsrs == cExitStoreMsrs);
2682 Assert(cExitStoreMsrs == cExitLoadMsrs);
2683 uint32_t const cMsrs = cExitLoadMsrs;
2684
2685 /* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2686 Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2687
2688 /* Verify the MSR counts are within the allocated page size. */
2689 Assert(sizeof(VMXAUTOMSR) * cMsrs <= X86_PAGE_4K_SIZE);
2690
2691 /* Verify the relevant contents of the MSR areas match. */
2692 PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2693 PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2694 PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2695 bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2696 for (uint32_t i = 0; i < cMsrs; i++)
2697 {
2698 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2699 if (fSeparateExitMsrStorePage)
2700 {
2701 AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2702 ("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2703 pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2704 }
2705
2706 AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2707 ("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2708 pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2709
2710 uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2711 AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2712 ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2713 pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2714
2715 /* Verify that cached host EFER MSR matches what's loaded the CPU. */
2716 bool const fIsEferMsr = RT_BOOL(pHostMsrLoad->u32Msr == MSR_K6_EFER);
2717 if (fIsEferMsr)
2718 {
2719 AssertMsgReturnVoid(u64Msr == pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer,
2720 ("Cached=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2721 pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer, u64Msr, cMsrs));
2722 }
2723
2724 /* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2725 if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2726 {
2727 uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2728 if (fIsEferMsr)
2729 {
2730 AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2731 AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2732 }
2733 else
2734 {
2735 if (!fIsNstGstVmcs)
2736 {
2737 AssertMsgReturnVoid((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR,
2738 ("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2739 }
2740 else
2741 {
2742 /*
2743 * A nested-guest VMCS must -also- allow read/write passthrough for the MSR for us to
2744 * execute a nested-guest with MSR passthrough.
2745 *
2746 * Check if the nested-guest MSR bitmap allows passthrough, and if so, assert that we
2747 * allow passthrough too.
2748 */
2749 void const *pvMsrBitmapNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
2750 Assert(pvMsrBitmapNstGst);
2751 uint32_t const fMsrpmNstGst = CPUMGetVmxMsrPermission(pvMsrBitmapNstGst, pGuestMsrLoad->u32Msr);
2752 AssertMsgReturnVoid(fMsrpm == fMsrpmNstGst,
2753 ("u32Msr=%#RX32 cMsrs=%u Permission mismatch fMsrpm=%#x fMsrpmNstGst=%#x!\n",
2754 pGuestMsrLoad->u32Msr, cMsrs, fMsrpm, fMsrpmNstGst));
2755 }
2756 }
2757 }
2758
2759 /* Move to the next MSR. */
2760 pHostMsrLoad++;
2761 pGuestMsrLoad++;
2762 pGuestMsrStore++;
2763 }
2764}
2765#endif /* VBOX_STRICT */
2766
2767
2768/**
2769 * Flushes the TLB using EPT.
2770 *
2771 * @returns VBox status code.
2772 * @param pVCpu The cross context virtual CPU structure of the calling
2773 * EMT. Can be NULL depending on @a enmTlbFlush.
2774 * @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2775 * enmTlbFlush.
2776 * @param enmTlbFlush Type of flush.
2777 *
2778 * @remarks Caller is responsible for making sure this function is called only
2779 * when NestedPaging is supported and providing @a enmTlbFlush that is
2780 * supported by the CPU.
2781 * @remarks Can be called with interrupts disabled.
2782 */
2783static void hmR0VmxFlushEpt(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2784{
2785 uint64_t au64Descriptor[2];
2786 if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2787 au64Descriptor[0] = 0;
2788 else
2789 {
2790 Assert(pVCpu);
2791 Assert(pVmcsInfo);
2792 au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2793 }
2794 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2795
2796 int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2797 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2798
2799 if ( RT_SUCCESS(rc)
2800 && pVCpu)
2801 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2802}
2803
2804
2805/**
2806 * Flushes the TLB using VPID.
2807 *
2808 * @returns VBox status code.
2809 * @param pVCpu The cross context virtual CPU structure of the calling
2810 * EMT. Can be NULL depending on @a enmTlbFlush.
2811 * @param enmTlbFlush Type of flush.
2812 * @param GCPtr Virtual address of the page to flush (can be 0 depending
2813 * on @a enmTlbFlush).
2814 *
2815 * @remarks Can be called with interrupts disabled.
2816 */
2817static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2818{
2819 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2820
2821 uint64_t au64Descriptor[2];
2822 if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2823 {
2824 au64Descriptor[0] = 0;
2825 au64Descriptor[1] = 0;
2826 }
2827 else
2828 {
2829 AssertPtr(pVCpu);
2830 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2831 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2832 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2833 au64Descriptor[1] = GCPtr;
2834 }
2835
2836 int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2837 AssertMsg(rc == VINF_SUCCESS,
2838 ("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2839
2840 if ( RT_SUCCESS(rc)
2841 && pVCpu)
2842 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2843 NOREF(rc);
2844}
2845
2846
2847/**
2848 * Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2849 * otherwise there is nothing really to invalidate.
2850 *
2851 * @returns VBox status code.
2852 * @param pVCpu The cross context virtual CPU structure.
2853 * @param GCVirt Guest virtual address of the page to invalidate.
2854 */
2855VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
2856{
2857 AssertPtr(pVCpu);
2858 LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2859
2860 if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2861 {
2862 /*
2863 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2864 * the EPT case. See @bugref{6043} and @bugref{6177}.
2865 *
2866 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2867 * as this function maybe called in a loop with individual addresses.
2868 */
2869 PVM pVM = pVCpu->CTX_SUFF(pVM);
2870 if (pVM->hm.s.vmx.fVpid)
2871 {
2872 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2873
2874#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2875 /*
2876 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
2877 * where executing INVVPID outside 64-bit mode does not flush translations of
2878 * 64-bit linear addresses, see @bugref{6208#c72}.
2879 */
2880 if (RT_HI_U32(GCVirt))
2881 fVpidFlush = false;
2882#endif
2883
2884 if (fVpidFlush)
2885 {
2886 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2887 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2888 }
2889 else
2890 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2891 }
2892 else if (pVM->hm.s.fNestedPaging)
2893 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2894 }
2895
2896 return VINF_SUCCESS;
2897}
2898
2899
2900/**
2901 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2902 * case where neither EPT nor VPID is supported by the CPU.
2903 *
2904 * @param pHostCpu The HM physical-CPU structure.
2905 * @param pVCpu The cross context virtual CPU structure.
2906 *
2907 * @remarks Called with interrupts disabled.
2908 */
2909static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2910{
2911 AssertPtr(pVCpu);
2912 AssertPtr(pHostCpu);
2913
2914 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2915
2916 Assert(pHostCpu->idCpu != NIL_RTCPUID);
2917 pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2918 pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2919 pVCpu->hm.s.fForceTLBFlush = false;
2920 return;
2921}
2922
2923
2924/**
2925 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2926 *
2927 * @param pHostCpu The HM physical-CPU structure.
2928 * @param pVCpu The cross context virtual CPU structure.
2929 * @param pVmcsInfo The VMCS info. object.
2930 *
2931 * @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2932 * nomenclature. The reason is, to avoid confusion in compare statements
2933 * since the host-CPU copies are named "ASID".
2934 *
2935 * @remarks Called with interrupts disabled.
2936 */
2937static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2938{
2939#ifdef VBOX_WITH_STATISTICS
2940 bool fTlbFlushed = false;
2941# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2942# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2943 if (!fTlbFlushed) \
2944 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2945 } while (0)
2946#else
2947# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2948# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2949#endif
2950
2951 AssertPtr(pVCpu);
2952 AssertPtr(pHostCpu);
2953 Assert(pHostCpu->idCpu != NIL_RTCPUID);
2954
2955 PVM pVM = pVCpu->CTX_SUFF(pVM);
2956 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2957 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2958 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2959
2960 /*
2961 * Force a TLB flush for the first world-switch if the current CPU differs from the one we
2962 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2963 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2964 * cannot reuse the current ASID anymore.
2965 */
2966 if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2967 || pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2968 {
2969 ++pHostCpu->uCurrentAsid;
2970 if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2971 {
2972 pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2973 pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2974 pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2975 }
2976
2977 pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2978 pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2979 pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2980
2981 /*
2982 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2983 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2984 */
2985 hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2986 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2987 HMVMX_SET_TAGGED_TLB_FLUSHED();
2988 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2989 }
2990 else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2991 {
2992 /*
2993 * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2994 * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2995 * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2996 * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2997 * mappings, see @bugref{6568}.
2998 *
2999 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
3000 */
3001 hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
3002 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3003 HMVMX_SET_TAGGED_TLB_FLUSHED();
3004 }
3005
3006 pVCpu->hm.s.fForceTLBFlush = false;
3007 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
3008
3009 Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
3010 Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
3011 AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
3012 ("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
3013 AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
3014 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
3015 pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
3016 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
3017 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
3018
3019 /* Update VMCS with the VPID. */
3020 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3021 AssertRC(rc);
3022
3023#undef HMVMX_SET_TAGGED_TLB_FLUSHED
3024}
3025
3026
3027/**
3028 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
3029 *
3030 * @param pHostCpu The HM physical-CPU structure.
3031 * @param pVCpu The cross context virtual CPU structure.
3032 * @param pVmcsInfo The VMCS info. object.
3033 *
3034 * @remarks Called with interrupts disabled.
3035 */
3036static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
3037{
3038 AssertPtr(pVCpu);
3039 AssertPtr(pHostCpu);
3040 Assert(pHostCpu->idCpu != NIL_RTCPUID);
3041 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
3042 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
3043
3044 /*
3045 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
3046 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
3047 */
3048 if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3049 || pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3050 {
3051 pVCpu->hm.s.fForceTLBFlush = true;
3052 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3053 }
3054
3055 /* Check for explicit TLB flushes. */
3056 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3057 {
3058 pVCpu->hm.s.fForceTLBFlush = true;
3059 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3060 }
3061
3062 pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3063 pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3064
3065 if (pVCpu->hm.s.fForceTLBFlush)
3066 {
3067 hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
3068 pVCpu->hm.s.fForceTLBFlush = false;
3069 }
3070}
3071
3072
3073/**
3074 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
3075 *
3076 * @param pHostCpu The HM physical-CPU structure.
3077 * @param pVCpu The cross context virtual CPU structure.
3078 *
3079 * @remarks Called with interrupts disabled.
3080 */
3081static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
3082{
3083 AssertPtr(pVCpu);
3084 AssertPtr(pHostCpu);
3085 Assert(pHostCpu->idCpu != NIL_RTCPUID);
3086 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
3087 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
3088
3089 /*
3090 * Force a TLB flush for the first world switch if the current CPU differs from the one we
3091 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
3092 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
3093 * cannot reuse the current ASID anymore.
3094 */
3095 if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3096 || pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3097 {
3098 pVCpu->hm.s.fForceTLBFlush = true;
3099 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3100 }
3101
3102 /* Check for explicit TLB flushes. */
3103 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3104 {
3105 /*
3106 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see
3107 * hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
3108 * fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
3109 * include fExplicitFlush's too) - an obscure corner case.
3110 */
3111 pVCpu->hm.s.fForceTLBFlush = true;
3112 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3113 }
3114
3115 PVM pVM = pVCpu->CTX_SUFF(pVM);
3116 pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3117 if (pVCpu->hm.s.fForceTLBFlush)
3118 {
3119 ++pHostCpu->uCurrentAsid;
3120 if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
3121 {
3122 pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
3123 pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
3124 pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
3125 }
3126
3127 pVCpu->hm.s.fForceTLBFlush = false;
3128 pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3129 pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
3130 if (pHostCpu->fFlushAsidBeforeUse)
3131 {
3132 if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
3133 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
3134 else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
3135 {
3136 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
3137 pHostCpu->fFlushAsidBeforeUse = false;
3138 }
3139 else
3140 {
3141 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
3142 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
3143 }
3144 }
3145 }
3146
3147 AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
3148 ("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
3149 AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
3150 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
3151 pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
3152 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
3153 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
3154
3155 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3156 AssertRC(rc);
3157}
3158
3159
3160/**
3161 * Flushes the guest TLB entry based on CPU capabilities.
3162 *
3163 * @param pHostCpu The HM physical-CPU structure.
3164 * @param pVCpu The cross context virtual CPU structure.
3165 * @param pVmcsInfo The VMCS info. object.
3166 *
3167 * @remarks Called with interrupts disabled.
3168 */
3169static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3170{
3171#ifdef HMVMX_ALWAYS_FLUSH_TLB
3172 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
3173#endif
3174 PVM pVM = pVCpu->CTX_SUFF(pVM);
3175 switch (pVM->hm.s.vmx.enmTlbFlushType)
3176 {
3177 case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
3178 case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
3179 case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
3180 case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
3181 default:
3182 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
3183 break;
3184 }
3185 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
3186}
3187
3188
3189/**
3190 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
3191 * TLB entries from the host TLB before VM-entry.
3192 *
3193 * @returns VBox status code.
3194 * @param pVM The cross context VM structure.
3195 */
3196static int hmR0VmxSetupTaggedTlb(PVM pVM)
3197{
3198 /*
3199 * Determine optimal flush type for nested paging.
3200 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
3201 * unrestricted guest execution (see hmR3InitFinalizeR0()).
3202 */
3203 if (pVM->hm.s.fNestedPaging)
3204 {
3205 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
3206 {
3207 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
3208 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
3209 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3210 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
3211 else
3212 {
3213 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
3214 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3215 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
3216 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3217 }
3218
3219 /* Make sure the write-back cacheable memory type for EPT is supported. */
3220 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
3221 {
3222 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3223 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
3224 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3225 }
3226
3227 /* EPT requires a page-walk length of 4. */
3228 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
3229 {
3230 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3231 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
3232 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3233 }
3234 }
3235 else
3236 {
3237 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
3238 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3239 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
3240 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3241 }
3242 }
3243
3244 /*
3245 * Determine optimal flush type for VPID.
3246 */
3247 if (pVM->hm.s.vmx.fVpid)
3248 {
3249 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
3250 {
3251 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
3252 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
3253 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
3254 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
3255 else
3256 {
3257 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
3258 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
3259 LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
3260 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
3261 LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
3262 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3263 pVM->hm.s.vmx.fVpid = false;
3264 }
3265 }
3266 else
3267 {
3268 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
3269 Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
3270 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3271 pVM->hm.s.vmx.fVpid = false;
3272 }
3273 }
3274
3275 /*
3276 * Setup the handler for flushing tagged-TLBs.
3277 */
3278 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
3279 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
3280 else if (pVM->hm.s.fNestedPaging)
3281 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
3282 else if (pVM->hm.s.vmx.fVpid)
3283 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
3284 else
3285 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
3286 return VINF_SUCCESS;
3287}
3288
3289
3290#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3291/**
3292 * Sets up the shadow VMCS fields arrays.
3293 *
3294 * This function builds arrays of VMCS fields to sync the shadow VMCS later while
3295 * executing the guest.
3296 *
3297 * @returns VBox status code.
3298 * @param pVM The cross context VM structure.
3299 */
3300static int hmR0VmxSetupShadowVmcsFieldsArrays(PVM pVM)
3301{
3302 /*
3303 * Paranoia. Ensure we haven't exposed the VMWRITE-All VMX feature to the guest
3304 * when the host does not support it.
3305 */
3306 bool const fGstVmwriteAll = pVM->cpum.ro.GuestFeatures.fVmxVmwriteAll;
3307 if ( !fGstVmwriteAll
3308 || (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL))
3309 { /* likely. */ }
3310 else
3311 {
3312 LogRelFunc(("VMX VMWRITE-All feature exposed to the guest but host CPU does not support it!\n"));
3313 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_GST_HOST_VMWRITE_ALL;
3314 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3315 }
3316
3317 uint32_t const cVmcsFields = RT_ELEMENTS(g_aVmcsFields);
3318 uint32_t cRwFields = 0;
3319 uint32_t cRoFields = 0;
3320 for (uint32_t i = 0; i < cVmcsFields; i++)
3321 {
3322 VMXVMCSFIELD VmcsField;
3323 VmcsField.u = g_aVmcsFields[i];
3324
3325 /*
3326 * We will be writing "FULL" (64-bit) fields while syncing the shadow VMCS.
3327 * Therefore, "HIGH" (32-bit portion of 64-bit) fields must not be included
3328 * in the shadow VMCS fields array as they would be redundant.
3329 *
3330 * If the VMCS field depends on a CPU feature that is not exposed to the guest,
3331 * we must not include it in the shadow VMCS fields array. Guests attempting to
3332 * VMREAD/VMWRITE such VMCS fields would cause a VM-exit and we shall emulate
3333 * the required behavior.
3334 */
3335 if ( VmcsField.n.fAccessType == VMX_VMCSFIELD_ACCESS_FULL
3336 && CPUMIsGuestVmxVmcsFieldValid(pVM, VmcsField.u))
3337 {
3338 /*
3339 * Read-only fields are placed in a separate array so that while syncing shadow
3340 * VMCS fields later (which is more performance critical) we can avoid branches.
3341 *
3342 * However, if the guest can write to all fields (including read-only fields),
3343 * we treat it a as read/write field. Otherwise, writing to these fields would
3344 * cause a VMWRITE instruction error while syncing the shadow VMCS .
3345 */
3346 if ( fGstVmwriteAll
3347 || !HMVmxIsVmcsFieldReadOnly(VmcsField.u))
3348 pVM->hm.s.vmx.paShadowVmcsFields[cRwFields++] = VmcsField.u;
3349 else
3350 pVM->hm.s.vmx.paShadowVmcsRoFields[cRoFields++] = VmcsField.u;
3351 }
3352 }
3353
3354 /* Update the counts. */
3355 pVM->hm.s.vmx.cShadowVmcsFields = cRwFields;
3356 pVM->hm.s.vmx.cShadowVmcsRoFields = cRoFields;
3357 return VINF_SUCCESS;
3358}
3359
3360
3361/**
3362 * Sets up the VMREAD and VMWRITE bitmaps.
3363 *
3364 * @param pVM The cross context VM structure.
3365 */
3366static void hmR0VmxSetupVmreadVmwriteBitmaps(PVM pVM)
3367{
3368 /*
3369 * By default, ensure guest attempts to acceses to any VMCS fields cause VM-exits.
3370 */
3371 uint32_t const cbBitmap = X86_PAGE_4K_SIZE;
3372 uint8_t *pbVmreadBitmap = (uint8_t *)pVM->hm.s.vmx.pvVmreadBitmap;
3373 uint8_t *pbVmwriteBitmap = (uint8_t *)pVM->hm.s.vmx.pvVmwriteBitmap;
3374 ASMMemFill32(pbVmreadBitmap, cbBitmap, UINT32_C(0xffffffff));
3375 ASMMemFill32(pbVmwriteBitmap, cbBitmap, UINT32_C(0xffffffff));
3376
3377 /*
3378 * Skip intercepting VMREAD/VMWRITE to guest read/write fields in the
3379 * VMREAD and VMWRITE bitmaps.
3380 */
3381 {
3382 uint32_t const *paShadowVmcsFields = pVM->hm.s.vmx.paShadowVmcsFields;
3383 uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
3384 for (uint32_t i = 0; i < cShadowVmcsFields; i++)
3385 {
3386 uint32_t const uVmcsField = paShadowVmcsFields[i];
3387 Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3388 Assert(uVmcsField >> 3 < cbBitmap);
3389 ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3390 ASMBitClear(pbVmwriteBitmap + (uVmcsField >> 3), uVmcsField & 7);
3391 }
3392 }
3393
3394 /*
3395 * Skip intercepting VMREAD for guest read-only fields in the VMREAD bitmap
3396 * if the host supports VMWRITE to all supported VMCS fields.
3397 */
3398 if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
3399 {
3400 uint32_t const *paShadowVmcsRoFields = pVM->hm.s.vmx.paShadowVmcsRoFields;
3401 uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
3402 for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
3403 {
3404 uint32_t const uVmcsField = paShadowVmcsRoFields[i];
3405 Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3406 Assert(uVmcsField >> 3 < cbBitmap);
3407 ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3408 }
3409 }
3410}
3411#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
3412
3413
3414/**
3415 * Sets up the virtual-APIC page address for the VMCS.
3416 *
3417 * @returns VBox status code.
3418 * @param pVCpu The cross context virtual CPU structure.
3419 * @param pVmcsInfo The VMCS info. object.
3420 */
3421DECLINLINE(int) hmR0VmxSetupVmcsVirtApicAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
3422{
3423 NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3424 RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
3425 Assert(HCPhysVirtApic != NIL_RTHCPHYS);
3426 Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
3427 return VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
3428}
3429
3430
3431/**
3432 * Sets up the MSR-bitmap address for the VMCS.
3433 *
3434 * @returns VBox status code.
3435 * @param pVCpu The cross context virtual CPU structure.
3436 * @param pVmcsInfo The VMCS info. object.
3437 */
3438DECLINLINE(int) hmR0VmxSetupVmcsMsrBitmapAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
3439{
3440 NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3441 RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
3442 Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
3443 Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3444 return VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
3445}
3446
3447
3448/**
3449 * Sets up the APIC-access page address for the VMCS.
3450 *
3451 * @returns VBox status code.
3452 * @param pVCpu The cross context virtual CPU structure.
3453 */
3454DECLINLINE(int) hmR0VmxSetupVmcsApicAccessAddr(PVMCPU pVCpu)
3455{
3456 RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
3457 Assert(HCPhysApicAccess != NIL_RTHCPHYS);
3458 Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
3459 return VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
3460}
3461
3462
3463#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3464/**
3465 * Sets up the VMREAD bitmap address for the VMCS.
3466 *
3467 * @returns VBox status code.
3468 * @param pVCpu The cross context virtual CPU structure.
3469 */
3470DECLINLINE(int) hmR0VmxSetupVmcsVmreadBitmapAddr(PVMCPU pVCpu)
3471{
3472 RTHCPHYS const HCPhysVmreadBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmreadBitmap;
3473 Assert(HCPhysVmreadBitmap != NIL_RTHCPHYS);
3474 Assert(!(HCPhysVmreadBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3475 return VMXWriteVmcs64(VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL, HCPhysVmreadBitmap);
3476}
3477
3478
3479/**
3480 * Sets up the VMWRITE bitmap address for the VMCS.
3481 *
3482 * @returns VBox status code.
3483 * @param pVCpu The cross context virtual CPU structure.
3484 */
3485DECLINLINE(int) hmR0VmxSetupVmcsVmwriteBitmapAddr(PVMCPU pVCpu)
3486{
3487 RTHCPHYS const HCPhysVmwriteBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmwriteBitmap;
3488 Assert(HCPhysVmwriteBitmap != NIL_RTHCPHYS);
3489 Assert(!(HCPhysVmwriteBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3490 return VMXWriteVmcs64(VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL, HCPhysVmwriteBitmap);
3491}
3492#endif
3493
3494
3495/**
3496 * Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
3497 * in the VMCS.
3498 *
3499 * @returns VBox status code.
3500 * @param pVCpu The cross context virtual CPU structure.
3501 * @param pVmcsInfo The VMCS info. object.
3502 */
3503DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3504{
3505 NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3506
3507 RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
3508 Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
3509 Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3510
3511 RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
3512 Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
3513 Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
3514
3515 RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
3516 Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
3517 Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3518
3519 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad);
3520 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore);
3521 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad);
3522 AssertRCReturn(rc, rc);
3523 return VINF_SUCCESS;
3524}
3525
3526
3527/**
3528 * Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
3529 *
3530 * @param pVCpu The cross context virtual CPU structure.
3531 * @param pVmcsInfo The VMCS info. object.
3532 * @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3533 */
3534static void hmR0VmxSetupVmcsMsrPermissions(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3535{
3536 Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
3537
3538 /*
3539 * The guest can access the following MSRs (read, write) without causing
3540 * VM-exits; they are loaded/stored automatically using fields in the VMCS.
3541 */
3542 PVM pVM = pVCpu->CTX_SUFF(pVM);
3543 hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
3544 hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
3545 hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
3546 hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3547 hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
3548
3549 /*
3550 * The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
3551 * associated with then. We never need to intercept access (writes need to be
3552 * executed without causing a VM-exit, reads will #GP fault anyway).
3553 *
3554 * The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
3555 * read/write them. We swap the the guest/host MSR value using the
3556 * auto-load/store MSR area.
3557 */
3558 if (pVM->cpum.ro.GuestFeatures.fIbpb)
3559 hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
3560 if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
3561 hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
3562 if (pVM->cpum.ro.GuestFeatures.fIbrs)
3563 hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
3564
3565#if HC_ARCH_BITS == 64
3566 /*
3567 * Allow full read/write access for the following MSRs (mandatory for VT-x)
3568 * required for 64-bit guests.
3569 */
3570 if (pVM->hm.s.fAllow64BitGuests)
3571 {
3572 hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
3573 hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
3574 hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
3575 hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3576 }
3577#endif
3578
3579 /*
3580 * IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
3581 */
3582#ifdef VBOX_STRICT
3583 Assert(pVmcsInfo->pvMsrBitmap);
3584 uint32_t const fMsrpmEfer = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
3585 Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
3586#endif
3587}
3588
3589
3590/**
3591 * Sets up pin-based VM-execution controls in the VMCS.
3592 *
3593 * @returns VBox status code.
3594 * @param pVCpu The cross context virtual CPU structure.
3595 * @param pVmcsInfo The VMCS info. object.
3596 */
3597static int hmR0VmxSetupVmcsPinCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3598{
3599 PVM pVM = pVCpu->CTX_SUFF(pVM);
3600 uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
3601 uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
3602
3603 fVal |= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
3604 | VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
3605
3606 if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
3607 fVal |= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
3608
3609 /* Enable the VMX-preemption timer. */
3610 if (pVM->hm.s.vmx.fUsePreemptTimer)
3611 {
3612 Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
3613 fVal |= VMX_PIN_CTLS_PREEMPT_TIMER;
3614 }
3615
3616#if 0
3617 /* Enable posted-interrupt processing. */
3618 if (pVM->hm.s.fPostedIntrs)
3619 {
3620 Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
3621 Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
3622 fVal |= VMX_PIN_CTLS_POSTED_INT;
3623 }
3624#endif
3625
3626 if ((fVal & fZap) != fVal)
3627 {
3628 LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3629 pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
3630 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
3631 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3632 }
3633
3634 /* Commit it to the VMCS and update our cache. */
3635 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
3636 AssertRCReturn(rc, rc);
3637 pVmcsInfo->u32PinCtls = fVal;
3638
3639 return VINF_SUCCESS;
3640}
3641
3642
3643/**
3644 * Sets up secondary processor-based VM-execution controls in the VMCS.
3645 *
3646 * @returns VBox status code.
3647 * @param pVCpu The cross context virtual CPU structure.
3648 * @param pVmcsInfo The VMCS info. object.
3649 */
3650static int hmR0VmxSetupVmcsProcCtls2(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3651{
3652 PVM pVM = pVCpu->CTX_SUFF(pVM);
3653 uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
3654 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3655
3656 /* WBINVD causes a VM-exit. */
3657 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
3658 fVal |= VMX_PROC_CTLS2_WBINVD_EXIT;
3659
3660 /* Enable EPT (aka nested-paging). */
3661 if (pVM->hm.s.fNestedPaging)
3662 fVal |= VMX_PROC_CTLS2_EPT;
3663
3664 /* Enable the INVPCID instruction if we expose it to the guest and is supported
3665 by the hardware. Without this, guest executing INVPCID would cause a #UD. */
3666 if ( pVM->cpum.ro.GuestFeatures.fInvpcid
3667 && (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID))
3668 fVal |= VMX_PROC_CTLS2_INVPCID;
3669
3670 /* Enable VPID. */
3671 if (pVM->hm.s.vmx.fVpid)
3672 fVal |= VMX_PROC_CTLS2_VPID;
3673
3674 /* Enable unrestricted guest execution. */
3675 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3676 fVal |= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
3677
3678#if 0
3679 if (pVM->hm.s.fVirtApicRegs)
3680 {
3681 /* Enable APIC-register virtualization. */
3682 Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
3683 fVal |= VMX_PROC_CTLS2_APIC_REG_VIRT;
3684
3685 /* Enable virtual-interrupt delivery. */
3686 Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3687 fVal |= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3688 }
3689#endif
3690
3691 /* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is
3692 where the TPR shadow resides. */
3693 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3694 * done dynamically. */
3695 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3696 {
3697 fVal |= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3698 int rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3699 AssertRCReturn(rc, rc);
3700 }
3701
3702 /* Enable the RDTSCP instruction if we expose it to the guest and is supported
3703 by the hardware. Without this, guest executing RDTSCP would cause a #UD. */
3704 if ( pVM->cpum.ro.GuestFeatures.fRdTscP
3705 && (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP))
3706 fVal |= VMX_PROC_CTLS2_RDTSCP;
3707
3708 /* Enable Pause-Loop exiting. */
3709 if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
3710 && pVM->hm.s.vmx.cPleGapTicks
3711 && pVM->hm.s.vmx.cPleWindowTicks)
3712 {
3713 fVal |= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3714
3715 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
3716 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
3717 AssertRCReturn(rc, rc);
3718 }
3719
3720 if ((fVal & fZap) != fVal)
3721 {
3722 LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3723 pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3724 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3725 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3726 }
3727
3728 /* Commit it to the VMCS and update our cache. */
3729 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3730 AssertRCReturn(rc, rc);
3731 pVmcsInfo->u32ProcCtls2 = fVal;
3732
3733 return VINF_SUCCESS;
3734}
3735
3736
3737/**
3738 * Sets up processor-based VM-execution controls in the VMCS.
3739 *
3740 * @returns VBox status code.
3741 * @param pVCpu The cross context virtual CPU structure.
3742 * @param pVmcsInfo The VMCS info. object.
3743 */
3744static int hmR0VmxSetupVmcsProcCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3745{
3746 PVM pVM = pVCpu->CTX_SUFF(pVM);
3747
3748 uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3749 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3750
3751 fVal |= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3752 | VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3753 | VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3754 | VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3755 | VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3756 | VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3757 | VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3758
3759 /* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3760 if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3761 || (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3762 {
3763 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3764 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3765 }
3766
3767 /* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3768 if (!pVM->hm.s.fNestedPaging)
3769 {
3770 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3771 fVal |= VMX_PROC_CTLS_INVLPG_EXIT
3772 | VMX_PROC_CTLS_CR3_LOAD_EXIT
3773 | VMX_PROC_CTLS_CR3_STORE_EXIT;
3774 }
3775
3776 /* Use TPR shadowing if supported by the CPU. */
3777 if ( PDMHasApic(pVM)
3778 && pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3779 {
3780 fVal |= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3781 /* CR8 writes cause a VM-exit based on TPR threshold. */
3782 Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3783 Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3784 int rc = hmR0VmxSetupVmcsVirtApicAddr(pVCpu, pVmcsInfo);
3785 AssertRCReturn(rc, rc);
3786 }
3787 else
3788 {
3789 /* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3790 invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3791 if (pVM->hm.s.fAllow64BitGuests)
3792 {
3793 fVal |= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3794 | VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3795 }
3796 }
3797
3798 /* Use MSR-bitmaps if supported by the CPU. */
3799 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3800 {
3801 fVal |= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3802 int rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3803 AssertRCReturn(rc, rc);
3804 }
3805
3806 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3807 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3808 fVal |= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3809
3810 if ((fVal & fZap) != fVal)
3811 {
3812 LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3813 pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3814 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3815 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3816 }
3817
3818 /* Commit it to the VMCS and update our cache. */
3819 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3820 AssertRCReturn(rc, rc);
3821 pVmcsInfo->u32ProcCtls = fVal;
3822
3823 /* Set up MSR permissions that don't change through the lifetime of the VM. */
3824 if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3825 hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo, false /* fIsNstGstVmcs */);
3826
3827 /* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3828 if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3829 return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3830
3831 /* Sanity check, should not really happen. */
3832 if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3833 { /* likely */ }
3834 else
3835 {
3836 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3837 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3838 }
3839
3840 /* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3841 return VINF_SUCCESS;
3842}
3843
3844
3845/**
3846 * Sets up miscellaneous (everything other than Pin, Processor and secondary
3847 * Processor-based VM-execution) control fields in the VMCS.
3848 *
3849 * @returns VBox status code.
3850 * @param pVCpu The cross context virtual CPU structure.
3851 * @param pVmcsInfo The VMCS info. object.
3852 */
3853static int hmR0VmxSetupVmcsMiscCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3854{
3855#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3856 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
3857 {
3858 int rc = hmR0VmxSetupVmcsVmreadBitmapAddr(pVCpu);
3859 rc |= hmR0VmxSetupVmcsVmwriteBitmapAddr(pVCpu);
3860 if (RT_SUCCESS(rc))
3861 { /* likely */ }
3862 else
3863 {
3864 LogRelFunc(("Failed to setup VMREAD/VMWRITE bitmap addresses. rc=%Rrc\n", rc));
3865 return rc;
3866 }
3867 }
3868#endif
3869
3870 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
3871 if (RT_SUCCESS(rc))
3872 {
3873 rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3874 if (RT_SUCCESS(rc))
3875 {
3876 uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3877 uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3878 rc = VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
3879 rc |= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
3880 if (RT_SUCCESS(rc))
3881 {
3882 pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3883 pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3884 return VINF_SUCCESS;
3885 }
3886 LogRelFunc(("Failed to initialize VMCS CR0/CR4 guest/host mask. rc=%Rrc\n", rc));
3887 }
3888 else
3889 LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3890 }
3891 else
3892 LogRelFunc(("Failed to initialize VMCS link pointer. rc=%Rrc\n", rc));
3893 return rc;
3894}
3895
3896
3897/**
3898 * Sets up the initial exception bitmap in the VMCS based on static conditions.
3899 *
3900 * We shall setup those exception intercepts that don't change during the
3901 * lifetime of the VM here. The rest are done dynamically while loading the
3902 * guest state.
3903 *
3904 * @returns VBox status code.
3905 * @param pVCpu The cross context virtual CPU structure.
3906 * @param pVmcsInfo The VMCS info. object.
3907 */
3908static int hmR0VmxSetupVmcsXcptBitmap(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3909{
3910 /*
3911 * The following exceptions are always intercepted:
3912 *
3913 * #AC - To prevent the guest from hanging the CPU.
3914 * #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3915 * recursive #DBs can cause a CPU hang.
3916 * #PF - To sync our shadow page tables when nested-paging is not used.
3917 */
3918 bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3919 uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3920 | RT_BIT(X86_XCPT_DB)
3921 | (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3922
3923 /* Commit it to the VMCS. */
3924 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3925 AssertRCReturn(rc, rc);
3926
3927 /* Update our cache of the exception bitmap. */
3928 pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3929 return VINF_SUCCESS;
3930}
3931
3932
3933#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3934/**
3935 * Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3936 *
3937 * @returns VBox status code.
3938 * @param pVCpu The cross context virtual CPU structure.
3939 * @param pVmcsInfo The VMCS info. object.
3940 */
3941static int hmR0VmxSetupVmcsCtlsNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3942{
3943 PVM pVM = pVCpu->CTX_SUFF(pVM);
3944 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
3945 if (RT_SUCCESS(rc))
3946 {
3947 rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3948 if (RT_SUCCESS(rc))
3949 {
3950 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3951 rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3952 if (RT_SUCCESS(rc))
3953 {
3954 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3955 rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3956 if (RT_SUCCESS(rc))
3957 return VINF_SUCCESS;
3958
3959 LogRelFunc(("Failed to set up the APIC-access address in the nested-guest VMCS. rc=%Rrc\n", rc));
3960 }
3961 else
3962 LogRelFunc(("Failed to set up the MSR-bitmap address in the nested-guest VMCS. rc=%Rrc\n", rc));
3963 }
3964 else
3965 LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3966 }
3967 else
3968 LogRelFunc(("Failed to set up the auto-load/store MSR addresses in the nested-guest VMCS. rc=%Rrc\n", rc));
3969
3970 return rc;
3971}
3972#endif
3973
3974
3975/**
3976 * Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3977 * VMX.
3978 *
3979 * @returns VBox status code.
3980 * @param pVCpu The cross context virtual CPU structure.
3981 * @param pVmcsInfo The VMCS info. object.
3982 * @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3983 */
3984static int hmR0VmxSetupVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3985{
3986 Assert(pVmcsInfo->pvVmcs);
3987 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3988
3989 /* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3990 PVM pVM = pVCpu->CTX_SUFF(pVM);
3991 *(uint32_t *)pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3992 const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3993
3994 LogFlowFunc(("\n"));
3995
3996 /*
3997 * Initialize the VMCS using VMCLEAR before loading the VMCS.
3998 * See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3999 */
4000 int rc = hmR0VmxClearVmcs(pVmcsInfo);
4001 if (RT_SUCCESS(rc))
4002 {
4003 rc = hmR0VmxLoadVmcs(pVmcsInfo);
4004 if (RT_SUCCESS(rc))
4005 {
4006 if (!fIsNstGstVmcs)
4007 {
4008 rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
4009 if (RT_SUCCESS(rc))
4010 {
4011 rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
4012 if (RT_SUCCESS(rc))
4013 {
4014 rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
4015 if (RT_SUCCESS(rc))
4016 {
4017 rc = hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
4018 if (RT_SUCCESS(rc))
4019 {
4020#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4021 /*
4022 * If a shadow VMCS is allocated for the VMCS info. object, initialize the
4023 * VMCS revision ID and shadow VMCS indicator bit. Also, clear the VMCS
4024 * making it fit for use when VMCS shadowing is later enabled.
4025 */
4026 if (pVmcsInfo->pvShadowVmcs)
4027 {
4028 VMXVMCSREVID VmcsRevId;
4029 VmcsRevId.u = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
4030 VmcsRevId.n.fIsShadowVmcs = 1;
4031 *(uint32_t *)pVmcsInfo->pvShadowVmcs = VmcsRevId.u;
4032 rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
4033 if (RT_SUCCESS(rc))
4034 { /* likely */ }
4035 else
4036 LogRelFunc(("Failed to initialize shadow VMCS. rc=%Rrc\n", rc));
4037 }
4038#endif
4039 }
4040 else
4041 LogRelFunc(("Failed to initialize exception bitmap. rc=%Rrc\n", rc));
4042 }
4043 else
4044 LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
4045 }
4046 else
4047 LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
4048 }
4049 else
4050 LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
4051 }
4052 else
4053 {
4054#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4055 rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
4056 if (RT_SUCCESS(rc))
4057 { /* likely */ }
4058 else
4059 LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
4060#else
4061 AssertFailed();
4062#endif
4063 }
4064 }
4065 else
4066 LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
4067 }
4068 else
4069 LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
4070
4071 /* Sync any CPU internal VMCS data back into our VMCS in memory. */
4072 if (RT_SUCCESS(rc))
4073 {
4074 rc = hmR0VmxClearVmcs(pVmcsInfo);
4075 if (RT_SUCCESS(rc))
4076 { /* likely */ }
4077 else
4078 LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
4079 }
4080
4081 /*
4082 * Update the last-error record both for failures and success, so we
4083 * can propagate the status code back to ring-3 for diagnostics.
4084 */
4085 hmR0VmxUpdateErrorRecord(pVCpu, rc);
4086 NOREF(pszVmcs);
4087 return rc;
4088}
4089
4090
4091/**
4092 * Does global VT-x initialization (called during module initialization).
4093 *
4094 * @returns VBox status code.
4095 */
4096VMMR0DECL(int) VMXR0GlobalInit(void)
4097{
4098#ifdef HMVMX_USE_FUNCTION_TABLE
4099 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
4100# ifdef VBOX_STRICT
4101 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
4102 Assert(g_apfnVMExitHandlers[i]);
4103# endif
4104#endif
4105 return VINF_SUCCESS;
4106}
4107
4108
4109/**
4110 * Does global VT-x termination (called during module termination).
4111 */
4112VMMR0DECL(void) VMXR0GlobalTerm()
4113{
4114 /* Nothing to do currently. */
4115}
4116
4117
4118/**
4119 * Sets up and activates VT-x on the current CPU.
4120 *
4121 * @returns VBox status code.
4122 * @param pHostCpu The HM physical-CPU structure.
4123 * @param pVM The cross context VM structure. Can be
4124 * NULL after a host resume operation.
4125 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
4126 * fEnabledByHost is @c true).
4127 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
4128 * @a fEnabledByHost is @c true).
4129 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
4130 * enable VT-x on the host.
4131 * @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
4132 */
4133VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
4134 PCSUPHWVIRTMSRS pHwvirtMsrs)
4135{
4136 AssertPtr(pHostCpu);
4137 AssertPtr(pHwvirtMsrs);
4138 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4139
4140 /* Enable VT-x if it's not already enabled by the host. */
4141 if (!fEnabledByHost)
4142 {
4143 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
4144 if (RT_FAILURE(rc))
4145 return rc;
4146 }
4147
4148 /*
4149 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
4150 * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
4151 * invalidated when flushing by VPID.
4152 */
4153 if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
4154 {
4155 hmR0VmxFlushEpt(NULL /* pVCpu */, NULL /* pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
4156 pHostCpu->fFlushAsidBeforeUse = false;
4157 }
4158 else
4159 pHostCpu->fFlushAsidBeforeUse = true;
4160
4161 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
4162 ++pHostCpu->cTlbFlushes;
4163
4164 return VINF_SUCCESS;
4165}
4166
4167
4168/**
4169 * Deactivates VT-x on the current CPU.
4170 *
4171 * @returns VBox status code.
4172 * @param pvCpuPage Pointer to the VMXON region.
4173 * @param HCPhysCpuPage Physical address of the VMXON region.
4174 *
4175 * @remarks This function should never be called when SUPR0EnableVTx() or
4176 * similar was used to enable VT-x on the host.
4177 */
4178VMMR0DECL(int) VMXR0DisableCpu(void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
4179{
4180 RT_NOREF2(pvCpuPage, HCPhysCpuPage);
4181
4182 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4183 return hmR0VmxLeaveRootMode();
4184}
4185
4186
4187/**
4188 * Does per-VM VT-x initialization.
4189 *
4190 * @returns VBox status code.
4191 * @param pVM The cross context VM structure.
4192 */
4193VMMR0DECL(int) VMXR0InitVM(PVM pVM)
4194{
4195 AssertPtr(pVM);
4196 LogFlowFunc(("pVM=%p\n", pVM));
4197
4198 int rc = hmR0VmxStructsAlloc(pVM);
4199 if (RT_FAILURE(rc))
4200 {
4201 LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
4202 return rc;
4203 }
4204
4205 return VINF_SUCCESS;
4206}
4207
4208
4209/**
4210 * Does per-VM VT-x termination.
4211 *
4212 * @returns VBox status code.
4213 * @param pVM The cross context VM structure.
4214 */
4215VMMR0DECL(int) VMXR0TermVM(PVM pVM)
4216{
4217 AssertPtr(pVM);
4218 LogFlowFunc(("pVM=%p\n", pVM));
4219
4220#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4221 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
4222 {
4223 Assert(pVM->hm.s.vmx.pvScratch);
4224 ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
4225 }
4226#endif
4227 hmR0VmxStructsFree(pVM);
4228 return VINF_SUCCESS;
4229}
4230
4231
4232/**
4233 * Sets up the VM for execution using hardware-assisted VMX.
4234 * This function is only called once per-VM during initialization.
4235 *
4236 * @returns VBox status code.
4237 * @param pVM The cross context VM structure.
4238 */
4239VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
4240{
4241 AssertPtr(pVM);
4242 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4243
4244 LogFlowFunc(("pVM=%p\n", pVM));
4245
4246 /*
4247 * At least verify if VMX is enabled, since we can't check if we're in
4248 * VMX root mode or not without causing a #GP.
4249 */
4250 RTCCUINTREG const uHostCr4 = ASMGetCR4();
4251 if (RT_LIKELY(uHostCr4 & X86_CR4_VMXE))
4252 { /* likely */ }
4253 else
4254 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
4255
4256 /*
4257 * Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable *must*
4258 * always be allocated. We no longer support the highly unlikely case of unrestricted guest
4259 * without pRealModeTSS, see hmR3InitFinalizeR0Intel().
4260 */
4261 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4262 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
4263 || !pVM->hm.s.vmx.pRealModeTSS))
4264 {
4265 LogRelFunc(("Invalid real-on-v86 state.\n"));
4266 return VERR_INTERNAL_ERROR;
4267 }
4268
4269 /* Initialize these always, see hmR3InitFinalizeR0().*/
4270 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
4271 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
4272
4273 /* Setup the tagged-TLB flush handlers. */
4274 int rc = hmR0VmxSetupTaggedTlb(pVM);
4275 if (RT_FAILURE(rc))
4276 {
4277 LogRelFunc(("Failed to setup tagged TLB. rc=%Rrc\n", rc));
4278 return rc;
4279 }
4280
4281#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4282 /* Setup the shadow VMCS fields array and VMREAD/VMWRITE bitmaps. */
4283 if (pVM->hm.s.vmx.fUseVmcsShadowing)
4284 {
4285 rc = hmR0VmxSetupShadowVmcsFieldsArrays(pVM);
4286 if (RT_SUCCESS(rc))
4287 hmR0VmxSetupVmreadVmwriteBitmaps(pVM);
4288 else
4289 {
4290 LogRelFunc(("Failed to setup shadow VMCS fields arrays. rc=%Rrc\n", rc));
4291 return rc;
4292 }
4293 }
4294#endif
4295
4296 for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
4297 {
4298 PVMCPU pVCpu = &pVM->aCpus[idCpu];
4299 Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
4300
4301 rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
4302 if (RT_SUCCESS(rc))
4303 {
4304#if HC_ARCH_BITS == 32
4305 hmR0VmxInitVmcsReadCache(pVCpu);
4306#endif
4307#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4308 if (pVM->cpum.ro.GuestFeatures.fVmx)
4309 {
4310 rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
4311 if (RT_SUCCESS(rc))
4312 { /* likely */ }
4313 else
4314 {
4315 LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
4316 return rc;
4317 }
4318 }
4319#endif
4320 }
4321 else
4322 {
4323 LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
4324 return rc;
4325 }
4326 }
4327
4328 return VINF_SUCCESS;
4329}
4330
4331
4332#if HC_ARCH_BITS == 32
4333# ifdef VBOX_ENABLE_64_BITS_GUESTS
4334/**
4335 * Check if guest state allows safe use of 32-bit switcher again.
4336 *
4337 * Segment bases and protected mode structures must be 32-bit addressable
4338 * because the 32-bit switcher will ignore high dword when writing these VMCS
4339 * fields. See @bugref{8432} for details.
4340 *
4341 * @returns true if safe, false if must continue to use the 64-bit switcher.
4342 * @param pCtx Pointer to the guest-CPU context.
4343 *
4344 * @remarks No-long-jump zone!!!
4345 */
4346static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
4347{
4348 if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4349 if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4350 if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4351 if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4352 if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4353 if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4354 if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4355 if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4356 if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4357 if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4358
4359 /* All good, bases are 32-bit. */
4360 return true;
4361}
4362# endif /* VBOX_ENABLE_64_BITS_GUESTS */
4363
4364# ifdef VBOX_STRICT
4365static bool hmR0VmxIsValidWriteFieldInCache(uint32_t idxField)
4366{
4367 switch (idxField)
4368 {
4369 case VMX_VMCS_GUEST_RIP:
4370 case VMX_VMCS_GUEST_RSP:
4371 case VMX_VMCS_GUEST_SYSENTER_EIP:
4372 case VMX_VMCS_GUEST_SYSENTER_ESP:
4373 case VMX_VMCS_GUEST_GDTR_BASE:
4374 case VMX_VMCS_GUEST_IDTR_BASE:
4375 case VMX_VMCS_GUEST_CS_BASE:
4376 case VMX_VMCS_GUEST_DS_BASE:
4377 case VMX_VMCS_GUEST_ES_BASE:
4378 case VMX_VMCS_GUEST_FS_BASE:
4379 case VMX_VMCS_GUEST_GS_BASE:
4380 case VMX_VMCS_GUEST_SS_BASE:
4381 case VMX_VMCS_GUEST_LDTR_BASE:
4382 case VMX_VMCS_GUEST_TR_BASE:
4383 case VMX_VMCS_GUEST_CR3:
4384 return true;
4385 }
4386 return false;
4387}
4388
4389static bool hmR0VmxIsValidReadFieldInCache(uint32_t idxField)
4390{
4391 switch (idxField)
4392 {
4393 /* Read-only fields. */
4394 case VMX_VMCS_RO_EXIT_QUALIFICATION:
4395 return true;
4396 }
4397 /* Remaining readable fields should also be writable. */
4398 return hmR0VmxIsValidWriteFieldInCache(idxField);
4399}
4400# endif /* VBOX_STRICT */
4401
4402
4403/**
4404 * Executes the specified handler in 64-bit mode.
4405 *
4406 * @returns VBox status code (no informational status codes).
4407 * @param pVCpu The cross context virtual CPU structure.
4408 * @param enmOp The operation to perform.
4409 * @param cParams Number of parameters.
4410 * @param paParam Array of 32-bit parameters.
4411 */
4412VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
4413{
4414 AssertPtr(pVCpu);
4415 PVM pVM = pVCpu->CTX_SUFF(pVM);
4416 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4417 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4418 Assert(pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Write.aField));
4419 Assert(pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Read.aField));
4420
4421#ifdef VBOX_STRICT
4422 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries; i++)
4423 Assert(hmR0VmxIsValidWriteFieldInCache(pVCpu->hm.s.vmx.VmcsCache.Write.aField[i]));
4424
4425 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries; i++)
4426 Assert(hmR0VmxIsValidReadFieldInCache(pVCpu->hm.s.vmx.VmcsCache.Read.aField[i]));
4427#endif
4428
4429 /* Disable interrupts. */
4430 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
4431
4432#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4433 RTCPUID idHostCpu = RTMpCpuId();
4434 CPUMR0SetLApic(pVCpu, idHostCpu);
4435#endif
4436
4437 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4438
4439 PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
4440 RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
4441
4442 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4443 PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
4444 hmR0VmxClearVmcs(pVmcsInfo);
4445
4446 /* Leave VMX root mode and disable VMX. */
4447 VMXDisable();
4448 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
4449
4450 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4451 CPUMSetHyperEIP(pVCpu, enmOp);
4452 for (int i = (int)cParams - 1; i >= 0; i--)
4453 CPUMPushHyper(pVCpu, paParam[i]);
4454
4455 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4456
4457 /* Call the switcher. */
4458 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
4459 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4460
4461 /* Re-enable VMX to make sure the VMX instructions don't cause #UD faults. */
4462 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
4463
4464 /* Re-enter VMX root mode. */
4465 int rc2 = VMXEnable(HCPhysCpuPage);
4466 if (RT_FAILURE(rc2))
4467 {
4468 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
4469 ASMSetFlags(fOldEFlags);
4470 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
4471 return rc2;
4472 }
4473
4474 /* Restore the VMCS as the current VMCS. */
4475 rc2 = hmR0VmxLoadVmcs(pVmcsInfo);
4476 AssertRC(rc2);
4477 Assert(!(ASMGetFlags() & X86_EFL_IF));
4478 ASMSetFlags(fOldEFlags);
4479 return rc;
4480}
4481
4482
4483/**
4484 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
4485 * supporting 64-bit guests.
4486 *
4487 * @returns VBox status code.
4488 * @param fResume Whether to VMLAUNCH or VMRESUME.
4489 * @param pCtx Pointer to the guest-CPU context.
4490 * @param pCache Pointer to the VMCS batch cache.
4491 * @param pVM The cross context VM structure.
4492 * @param pVCpu The cross context virtual CPU structure.
4493 */
4494DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMXVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4495{
4496 NOREF(fResume);
4497
4498 PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
4499 PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
4500 RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
4501
4502#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4503 pCache->uPos = 1;
4504 pCache->interPD = PGMGetInterPaeCR3(pVM);
4505 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4506#endif
4507
4508#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4509 pCache->TestIn.HCPhysCpuPage = 0;
4510 pCache->TestIn.HCPhysVmcs = 0;
4511 pCache->TestIn.pCache = 0;
4512 pCache->TestOut.HCPhysVmcs = 0;
4513 pCache->TestOut.pCache = 0;
4514 pCache->TestOut.pCtx = 0;
4515 pCache->TestOut.eflags = 0;
4516#else
4517 NOREF(pCache);
4518#endif
4519
4520 uint32_t aParam[10];
4521 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4522 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
4523 aParam[2] = RT_LO_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4524 aParam[3] = RT_HI_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
4525 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache);
4526 aParam[5] = 0;
4527 aParam[6] = VM_RC_ADDR(pVM, pVM);
4528 aParam[7] = 0;
4529 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
4530 aParam[9] = 0;
4531
4532#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4533 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4534 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4535#endif
4536 int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
4537
4538#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4539 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4540 Assert(pCtx->dr[4] == 10);
4541 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4542#endif
4543
4544#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4545 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4546 AssertMsg(pCache->TestIn.HCPhysVmcs == pVmcsInfo->HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4547 pVmcsInfo->HCPhysVmcs));
4548 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4549 pCache->TestOut.HCPhysVmcs));
4550 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4551 pCache->TestOut.pCache));
4552 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache),
4553 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache)));
4554 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4555 pCache->TestOut.pCtx));
4556 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4557#endif
4558 NOREF(pCtx);
4559 return rc;
4560}
4561#endif
4562
4563
4564/**
4565 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
4566 * the VMCS.
4567 *
4568 * @returns VBox status code.
4569 */
4570static int hmR0VmxExportHostControlRegs(void)
4571{
4572 RTCCUINTREG uReg = ASMGetCR0();
4573 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
4574 AssertRCReturn(rc, rc);
4575
4576 uReg = ASMGetCR3();
4577 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
4578 AssertRCReturn(rc, rc);
4579
4580 uReg = ASMGetCR4();
4581 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
4582 AssertRCReturn(rc, rc);
4583 return rc;
4584}
4585
4586
4587/**
4588 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
4589 * the host-state area in the VMCS.
4590 *
4591 * @returns VBox status code.
4592 * @param pVCpu The cross context virtual CPU structure.
4593 */
4594static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
4595{
4596#if HC_ARCH_BITS == 64
4597/**
4598 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
4599 * requirements. See hmR0VmxExportHostSegmentRegs().
4600 */
4601# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
4602 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
4603 { \
4604 bool fValidSelector = true; \
4605 if ((selValue) & X86_SEL_LDT) \
4606 { \
4607 uint32_t uAttr = ASMGetSegAttr((selValue)); \
4608 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
4609 } \
4610 if (fValidSelector) \
4611 { \
4612 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
4613 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
4614 } \
4615 (selValue) = 0; \
4616 }
4617
4618 /*
4619 * If we've executed guest code using hardware-assisted VMX, the host-state bits
4620 * will be messed up. We should -not- save the messed up state without restoring
4621 * the original host-state, see @bugref{7240}.
4622 *
4623 * This apparently can happen (most likely the FPU changes), deal with it rather than
4624 * asserting. Was observed booting Solaris 10u10 32-bit guest.
4625 */
4626 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
4627 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
4628 {
4629 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
4630 pVCpu->idCpu));
4631 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
4632 }
4633 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
4634#else
4635 RT_NOREF(pVCpu);
4636#endif
4637
4638 /*
4639 * Host DS, ES, FS and GS segment registers.
4640 */
4641#if HC_ARCH_BITS == 64
4642 RTSEL uSelDS = ASMGetDS();
4643 RTSEL uSelES = ASMGetES();
4644 RTSEL uSelFS = ASMGetFS();
4645 RTSEL uSelGS = ASMGetGS();
4646#else
4647 RTSEL uSelDS = 0;
4648 RTSEL uSelES = 0;
4649 RTSEL uSelFS = 0;
4650 RTSEL uSelGS = 0;
4651#endif
4652
4653 /*
4654 * Host CS and SS segment registers.
4655 */
4656 RTSEL uSelCS = ASMGetCS();
4657 RTSEL uSelSS = ASMGetSS();
4658
4659 /*
4660 * Host TR segment register.
4661 */
4662 RTSEL uSelTR = ASMGetTR();
4663
4664#if HC_ARCH_BITS == 64
4665 /*
4666 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
4667 * gain VM-entry and restore them before we get preempted.
4668 *
4669 * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
4670 */
4671 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
4672 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
4673 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
4674 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
4675# undef VMXLOCAL_ADJUST_HOST_SEG
4676#endif
4677
4678 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
4679 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
4680 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
4681 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
4682 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
4683 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
4684 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
4685 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
4686 Assert(uSelCS);
4687 Assert(uSelTR);
4688
4689 /* Write these host selector fields into the host-state area in the VMCS. */
4690 int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
4691 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
4692#if HC_ARCH_BITS == 64
4693 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
4694 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
4695 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
4696 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
4697#else
4698 NOREF(uSelDS);
4699 NOREF(uSelES);
4700 NOREF(uSelFS);
4701 NOREF(uSelGS);
4702#endif
4703 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
4704 AssertRCReturn(rc, rc);
4705
4706 /*
4707 * Host GDTR and IDTR.
4708 */
4709 RTGDTR Gdtr;
4710 RTIDTR Idtr;
4711 RT_ZERO(Gdtr);
4712 RT_ZERO(Idtr);
4713 ASMGetGDTR(&Gdtr);
4714 ASMGetIDTR(&Idtr);
4715 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
4716 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
4717 AssertRCReturn(rc, rc);
4718
4719#if HC_ARCH_BITS == 64
4720 /*
4721 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
4722 * them to the maximum limit (0xffff) on every VM-exit.
4723 */
4724 if (Gdtr.cbGdt != 0xffff)
4725 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
4726
4727 /*
4728 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4729 * Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4730 * as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4731 * However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4732 * uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4733 * alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4734 * at 0xffff on hosts where we are sure it won't cause trouble.
4735 */
4736# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
4737 if (Idtr.cbIdt < 0x0fff)
4738# else
4739 if (Idtr.cbIdt != 0xffff)
4740# endif
4741 {
4742 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
4743 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4744 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4745 }
4746#endif
4747
4748 /*
4749 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4750 * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4751 * RPL should be too in most cases.
4752 */
4753 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4754 ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4755
4756 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4757#if HC_ARCH_BITS == 64
4758 uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4759
4760 /*
4761 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4762 * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4763 * restoration if the host has something else. Task switching is not supported in 64-bit
4764 * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4765 * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4766 *
4767 * [1] See Intel spec. 3.5 "System Descriptor Types".
4768 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4769 */
4770 PVM pVM = pVCpu->CTX_SUFF(pVM);
4771 Assert(pDesc->System.u4Type == 11);
4772 if ( pDesc->System.u16LimitLow != 0x67
4773 || pDesc->System.u4LimitHigh)
4774 {
4775 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
4776 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4777 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4778 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
4779 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4780 }
4781
4782 /*
4783 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4784 */
4785 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
4786 {
4787 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4788 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4789 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4790 {
4791 /* The GDT is read-only but the writable GDT is available. */
4792 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4793 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4794 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4795 AssertRCReturn(rc, rc);
4796 }
4797 }
4798#else
4799 uintptr_t const uTRBase = X86DESC_BASE(pDesc);
4800#endif
4801 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
4802 AssertRCReturn(rc, rc);
4803
4804 /*
4805 * Host FS base and GS base.
4806 */
4807#if HC_ARCH_BITS == 64
4808 uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4809 uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4810 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
4811 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
4812 AssertRCReturn(rc, rc);
4813
4814 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4815 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4816 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4817 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4818 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4819#endif
4820 return VINF_SUCCESS;
4821}
4822
4823
4824/**
4825 * Exports certain host MSRs in the VM-exit MSR-load area and some in the
4826 * host-state area of the VMCS.
4827 *
4828 * These MSRs will be automatically restored on the host after every successful
4829 * VM-exit.
4830 *
4831 * @returns VBox status code.
4832 * @param pVCpu The cross context virtual CPU structure.
4833 *
4834 * @remarks No-long-jump zone!!!
4835 */
4836static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
4837{
4838 AssertPtr(pVCpu);
4839
4840 /*
4841 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4842 * rather than swapping them on every VM-entry.
4843 */
4844 hmR0VmxLazySaveHostMsrs(pVCpu);
4845
4846 /*
4847 * Host Sysenter MSRs.
4848 */
4849 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
4850#if HC_ARCH_BITS == 32
4851 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
4852 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
4853#else
4854 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
4855 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
4856#endif
4857 AssertRCReturn(rc, rc);
4858
4859 /*
4860 * Host EFER MSR.
4861 *
4862 * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4863 * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4864 */
4865 PVM pVM = pVCpu->CTX_SUFF(pVM);
4866 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4867 {
4868 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4869 AssertRCReturn(rc, rc);
4870 }
4871
4872 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4873 * hmR0VmxExportGuestEntryExitCtls(). */
4874
4875 return VINF_SUCCESS;
4876}
4877
4878
4879/**
4880 * Figures out if we need to swap the EFER MSR which is particularly expensive.
4881 *
4882 * We check all relevant bits. For now, that's everything besides LMA/LME, as
4883 * these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4884 *
4885 * @returns true if we need to load guest EFER, false otherwise.
4886 * @param pVCpu The cross context virtual CPU structure.
4887 *
4888 * @remarks Requires EFER, CR4.
4889 * @remarks No-long-jump zone!!!
4890 */
4891static bool hmR0VmxShouldSwapEferMsr(PCVMCPU pVCpu)
4892{
4893#ifdef HMVMX_ALWAYS_SWAP_EFER
4894 RT_NOREF(pVCpu);
4895 return true;
4896#else
4897 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4898#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4899 /* For 32-bit hosts running 64-bit guests, we always swap EFER MSR in the world-switcher. Nothing to do here. */
4900 if (CPUMIsGuestInLongModeEx(pCtx))
4901 return false;
4902#endif
4903
4904 PVM pVM = pVCpu->CTX_SUFF(pVM);
4905 uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4906 uint64_t const u64GuestEfer = pCtx->msrEFER;
4907
4908 /*
4909 * For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4910 * to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4911 */
4912 if ( CPUMIsGuestInLongModeEx(pCtx)
4913 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4914 return true;
4915
4916 /*
4917 * If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4918 * as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4919 *
4920 * See Intel spec. 4.5 "IA-32e Paging".
4921 * See Intel spec. 4.1.1 "Three Paging Modes".
4922 *
4923 * Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4924 * import CR4 and CR0 from the VMCS here as those bits are always up to date.
4925 */
4926 Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4927 Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4928 if ( (pCtx->cr4 & X86_CR4_PAE)
4929 && (pCtx->cr0 & X86_CR0_PG)
4930 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4931 {
4932 /* Assert that host is NX capable. */
4933 Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4934 return true;
4935 }
4936
4937 return false;
4938#endif
4939}
4940
4941/**
4942 * Exports the guest state with appropriate VM-entry and VM-exit controls in the
4943 * VMCS.
4944 *
4945 * This is typically required when the guest changes paging mode.
4946 *
4947 * @returns VBox status code.
4948 * @param pVCpu The cross context virtual CPU structure.
4949 * @param pVmxTransient The VMX-transient structure.
4950 *
4951 * @remarks Requires EFER.
4952 * @remarks No-long-jump zone!!!
4953 */
4954static int hmR0VmxExportGuestEntryExitCtls(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4955{
4956 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4957 {
4958 PVM pVM = pVCpu->CTX_SUFF(pVM);
4959 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4960
4961 /*
4962 * VM-entry controls.
4963 */
4964 {
4965 uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4966 uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4967
4968 /*
4969 * Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4970 * The first VT-x capable CPUs only supported the 1-setting of this bit.
4971 *
4972 * For nested-guests, this is a mandatory VM-entry control. It's also
4973 * required because we do not want to leak host bits to the nested-guest.
4974 */
4975 fVal |= VMX_ENTRY_CTLS_LOAD_DEBUG;
4976
4977 /*
4978 * Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4979 *
4980 * For nested-guests, the "IA-32e mode guest" control we initialize with what is
4981 * required to get the nested-guest working with hardware-assisted VMX execution.
4982 * It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a guest hypervisor
4983 * can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4984 * here rather than while merging the guest VMCS controls.
4985 */
4986 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4987 fVal |= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4988 else
4989 Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4990
4991 /*
4992 * If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4993 *
4994 * For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4995 * regardless of whether the nested-guest VMCS specifies it because we are free to
4996 * load whatever MSRs we require and we do not need to modify the guest visible copy
4997 * of the VM-entry MSR load area.
4998 */
4999 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
5000 && hmR0VmxShouldSwapEferMsr(pVCpu))
5001 fVal |= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
5002 else
5003 Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
5004
5005 /*
5006 * The following should -not- be set (since we're not in SMM mode):
5007 * - VMX_ENTRY_CTLS_ENTRY_TO_SMM
5008 * - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
5009 */
5010
5011 /** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
5012 * VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
5013
5014 if ((fVal & fZap) == fVal)
5015 { /* likely */ }
5016 else
5017 {
5018 Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
5019 pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
5020 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
5021 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
5022 }
5023
5024 /* Commit it to the VMCS. */
5025 if (pVmcsInfo->u32EntryCtls != fVal)
5026 {
5027 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
5028 AssertRCReturn(rc, rc);
5029 pVmcsInfo->u32EntryCtls = fVal;
5030 }
5031 }
5032
5033 /*
5034 * VM-exit controls.
5035 */
5036 {
5037 uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
5038 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
5039
5040 /*
5041 * Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
5042 * supported the 1-setting of this bit.
5043 *
5044 * For nested-guests, we set the "save debug controls" as the converse
5045 * "load debug controls" is mandatory for nested-guests anyway.
5046 */
5047 fVal |= VMX_EXIT_CTLS_SAVE_DEBUG;
5048
5049 /*
5050 * Set the host long mode active (EFER.LMA) bit (which Intel calls
5051 * "Host address-space size") if necessary. On VM-exit, VT-x sets both the
5052 * host EFER.LMA and EFER.LME bit to this value. See assertion in
5053 * hmR0VmxExportHostMsrs().
5054 *
5055 * For nested-guests, we always set this bit as we do not support 32-bit
5056 * hosts.
5057 */
5058#if HC_ARCH_BITS == 64
5059 fVal |= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
5060#else
5061 Assert(!pVmxTransient->fIsNestedGuest);
5062 Assert( pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64
5063 || pVmcsInfo->pfnStartVM == VMXR0StartVM32);
5064 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
5065 if (pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64)
5066 {
5067 /* The switcher returns to long mode, the EFER MSR is managed by the switcher. */
5068 fVal |= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
5069 }
5070 else
5071 Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
5072#endif
5073
5074 /*
5075 * If the VMCS EFER MSR fields are supported by the hardware, we use it.
5076 *
5077 * For nested-guests, we should use the "save IA32_EFER" control if we also
5078 * used the "load IA32_EFER" control while exporting VM-entry controls.
5079 */
5080 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
5081 && hmR0VmxShouldSwapEferMsr(pVCpu))
5082 {
5083 fVal |= VMX_EXIT_CTLS_SAVE_EFER_MSR
5084 | VMX_EXIT_CTLS_LOAD_EFER_MSR;
5085 }
5086
5087 /*
5088 * Enable saving of the VMX-preemption timer value on VM-exit.
5089 * For nested-guests, currently not exposed/used.
5090 */
5091 if ( pVM->hm.s.vmx.fUsePreemptTimer
5092 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
5093 fVal |= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
5094
5095 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
5096 Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
5097
5098 /** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
5099 * VMX_EXIT_CTLS_SAVE_PAT_MSR,
5100 * VMX_EXIT_CTLS_LOAD_PAT_MSR. */
5101
5102 if ((fVal & fZap) == fVal)
5103 { /* likely */ }
5104 else
5105 {
5106 Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
5107 pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
5108 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
5109 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
5110 }
5111
5112 /* Commit it to the VMCS. */
5113 if (pVmcsInfo->u32ExitCtls != fVal)
5114 {
5115 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
5116 AssertRCReturn(rc, rc);
5117 pVmcsInfo->u32ExitCtls = fVal;
5118 }
5119 }
5120
5121 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
5122 }
5123 return VINF_SUCCESS;
5124}
5125
5126
5127/**
5128 * Sets the TPR threshold in the VMCS.
5129 *
5130 * @returns VBox status code.
5131 * @param pVCpu The cross context virtual CPU structure.
5132 * @param pVmcsInfo The VMCS info. object.
5133 * @param u32TprThreshold The TPR threshold (task-priority class only).
5134 */
5135DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
5136{
5137 Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
5138 Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
5139 RT_NOREF2(pVCpu, pVmcsInfo);
5140 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
5141}
5142
5143
5144/**
5145 * Exports the guest APIC TPR state into the VMCS.
5146 *
5147 * @returns VBox status code.
5148 * @param pVCpu The cross context virtual CPU structure.
5149 * @param pVmxTransient The VMX-transient structure.
5150 *
5151 * @remarks No-long-jump zone!!!
5152 */
5153static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5154{
5155 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
5156 {
5157 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
5158
5159 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5160 if (!pVmxTransient->fIsNestedGuest)
5161 {
5162 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
5163 && APICIsEnabled(pVCpu))
5164 {
5165 /*
5166 * Setup TPR shadowing.
5167 */
5168 if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
5169 {
5170 bool fPendingIntr = false;
5171 uint8_t u8Tpr = 0;
5172 uint8_t u8PendingIntr = 0;
5173 int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
5174 AssertRCReturn(rc, rc);
5175
5176 /*
5177 * If there are interrupts pending but masked by the TPR, instruct VT-x to
5178 * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
5179 * priority of the pending interrupt so we can deliver the interrupt. If there
5180 * are no interrupts pending, set threshold to 0 to not cause any
5181 * TPR-below-threshold VM-exits.
5182 */
5183 Assert(pVmcsInfo->pbVirtApic);
5184 pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
5185 uint32_t u32TprThreshold = 0;
5186 if (fPendingIntr)
5187 {
5188 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
5189 (which is the Task-Priority Class). */
5190 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
5191 const uint8_t u8TprPriority = u8Tpr >> 4;
5192 if (u8PendingPriority <= u8TprPriority)
5193 u32TprThreshold = u8PendingPriority;
5194 }
5195
5196 rc = hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
5197 AssertRCReturn(rc, rc);
5198 }
5199 }
5200 }
5201 /* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
5202 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
5203 }
5204 return VINF_SUCCESS;
5205}
5206
5207
5208/**
5209 * Gets the guest interruptibility-state.
5210 *
5211 * @returns Guest's interruptibility-state.
5212 * @param pVCpu The cross context virtual CPU structure.
5213 * @param pVmxTransient The VMX-transient structure.
5214 *
5215 * @remarks No-long-jump zone!!!
5216 */
5217static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5218{
5219 /*
5220 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
5221 */
5222 uint32_t fIntrState = 0;
5223 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5224 {
5225 /* If inhibition is active, RIP and RFLAGS should've been updated
5226 (i.e. read previously from the VMCS or from ring-3). */
5227 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5228#ifdef VBOX_STRICT
5229 uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
5230 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5231 AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
5232#endif
5233 if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
5234 {
5235 if (pCtx->eflags.Bits.u1IF)
5236 fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
5237 else
5238 fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
5239 }
5240 else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5241 {
5242 /*
5243 * We can clear the inhibit force flag as even if we go back to the recompiler
5244 * without executing guest code in VT-x, the flag's condition to be cleared is
5245 * met and thus the cleared state is correct.
5246 */
5247 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5248 }
5249 }
5250
5251 /*
5252 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
5253 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
5254 * setting this would block host-NMIs and IRET will not clear the blocking.
5255 *
5256 * We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
5257 *
5258 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
5259 */
5260 if ( hmR0VmxIsPinCtlsSet(pVCpu, pVmxTransient, VMX_PIN_CTLS_VIRT_NMI)
5261 && CPUMIsGuestNmiBlocking(pVCpu))
5262 fIntrState |= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
5263
5264 return fIntrState;
5265}
5266
5267
5268/**
5269 * Exports the exception intercepts required for guest execution in the VMCS.
5270 *
5271 * @returns VBox status code.
5272 * @param pVCpu The cross context virtual CPU structure.
5273 * @param pVmxTransient The VMX-transient structure.
5274 *
5275 * @remarks No-long-jump zone!!!
5276 */
5277static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5278{
5279 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
5280 {
5281 /* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
5282 if ( !pVmxTransient->fIsNestedGuest
5283 && pVCpu->hm.s.fGIMTrapXcptUD)
5284 hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
5285 else
5286 hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
5287
5288 /* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
5289 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
5290 }
5291 return VINF_SUCCESS;
5292}
5293
5294
5295/**
5296 * Exports the guest's RIP into the guest-state area in the VMCS.
5297 *
5298 * @returns VBox status code.
5299 * @param pVCpu The cross context virtual CPU structure.
5300 *
5301 * @remarks No-long-jump zone!!!
5302 */
5303static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
5304{
5305 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
5306 {
5307 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
5308
5309 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
5310 AssertRCReturn(rc, rc);
5311
5312 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
5313 Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
5314 }
5315 return VINF_SUCCESS;
5316}
5317
5318
5319/**
5320 * Exports the guest's RSP into the guest-state area in the VMCS.
5321 *
5322 * @returns VBox status code.
5323 * @param pVCpu The cross context virtual CPU structure.
5324 *
5325 * @remarks No-long-jump zone!!!
5326 */
5327static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
5328{
5329 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
5330 {
5331 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
5332
5333 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
5334 AssertRCReturn(rc, rc);
5335
5336 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
5337 }
5338 return VINF_SUCCESS;
5339}
5340
5341
5342/**
5343 * Exports the guest's RFLAGS into the guest-state area in the VMCS.
5344 *
5345 * @returns VBox status code.
5346 * @param pVCpu The cross context virtual CPU structure.
5347 * @param pVmxTransient The VMX-transient structure.
5348 *
5349 * @remarks No-long-jump zone!!!
5350 */
5351static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5352{
5353 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
5354 {
5355 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
5356
5357 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
5358 Let us assert it as such and use 32-bit VMWRITE. */
5359 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
5360 X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
5361 Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
5362 Assert(!(fEFlags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
5363
5364 /*
5365 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
5366 * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
5367 * can run the real-mode guest code under Virtual 8086 mode.
5368 */
5369 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5370 if (pVmcsInfo->RealMode.fRealOnV86Active)
5371 {
5372 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5373 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5374 Assert(!pVmxTransient->fIsNestedGuest);
5375 pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
5376 fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
5377 fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
5378 }
5379
5380 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
5381 AssertRCReturn(rc, rc);
5382
5383 /*
5384 * Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
5385 *
5386 * We must avoid setting any automatic debug exceptions delivery when single-stepping
5387 * through the hypervisor debugger using EFLAGS.TF.
5388 */
5389 if ( !pVmxTransient->fIsNestedGuest
5390 && !pVCpu->hm.s.fSingleInstruction
5391 && fEFlags.Bits.u1TF)
5392 {
5393 /** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
5394 * premature trips to ring-3 esp since IEM does not yet handle it. */
5395 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
5396 AssertRCReturn(rc, rc);
5397 }
5398 /** @todo NSTVMX: Handling copying of VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS from
5399 * nested-guest VMCS. */
5400
5401 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
5402 Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
5403 }
5404 return VINF_SUCCESS;
5405}
5406
5407
5408#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5409/**
5410 * Copies the nested-guest VMCS to the shadow VMCS.
5411 *
5412 * @returns VBox status code.
5413 * @param pVCpu The cross context virtual CPU structure.
5414 * @param pVmcsInfo The VMCS info. object.
5415 *
5416 * @remarks No-long-jump zone!!!
5417 */
5418static int hmR0VmxCopyNstGstToShadowVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5419{
5420 PVM pVM = pVCpu->CTX_SUFF(pVM);
5421 PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5422
5423 /*
5424 * Disable interrupts so we don't get preempted while the shadow VMCS is the
5425 * current VMCS, as we may try saving guest lazy MSRs.
5426 *
5427 * Strictly speaking the lazy MSRs are not in the VMCS, but I'd rather not risk
5428 * calling the import VMCS code which is currently performing the guest MSR reads
5429 * (on 64-bit hosts) and accessing the auto-load/store MSR area on 32-bit hosts
5430 * and the rest of the VMX leave session machinery.
5431 */
5432 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
5433
5434 int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5435 if (RT_SUCCESS(rc))
5436 {
5437 /*
5438 * Copy all guest read/write VMCS fields.
5439 *
5440 * We don't check for VMWRITE failures here for performance reasons and
5441 * because they are not expected to fail, barring irrecoverable conditions
5442 * like hardware errors.
5443 */
5444 uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5445 for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5446 {
5447 uint64_t u64Val;
5448 uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5449 IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5450 VMXWriteVmcs64(uVmcsField, u64Val);
5451 }
5452
5453 /*
5454 * If the host CPU supports writing all VMCS fields, copy the guest read-only
5455 * VMCS fields, so the guest can VMREAD them without causing a VM-exit.
5456 */
5457 if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
5458 {
5459 uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
5460 for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
5461 {
5462 uint64_t u64Val;
5463 uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsRoFields[i];
5464 IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5465 VMXWriteVmcs64(uVmcsField, u64Val);
5466 }
5467 }
5468
5469 rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5470 rc |= hmR0VmxLoadVmcs(pVmcsInfo);
5471 }
5472
5473 ASMSetFlags(fEFlags);
5474 return rc;
5475}
5476
5477
5478/**
5479 * Copies the shadow VMCS to the nested-guest VMCS.
5480 *
5481 * @returns VBox status code.
5482 * @param pVCpu The cross context virtual CPU structure.
5483 * @param pVmcsInfo The VMCS info. object.
5484 *
5485 * @remarks Called with interrupts disabled.
5486 */
5487static int hmR0VmxCopyShadowToNstGstVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5488{
5489 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5490 PVM pVM = pVCpu->CTX_SUFF(pVM);
5491 PVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5492
5493 int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5494 if (RT_SUCCESS(rc))
5495 {
5496 /*
5497 * Copy guest read/write fields from the shadow VMCS.
5498 * Guest read-only fields cannot be modified, so no need to copy them.
5499 *
5500 * We don't check for VMREAD failures here for performance reasons and
5501 * because they are not expected to fail, barring irrecoverable conditions
5502 * like hardware errors.
5503 */
5504 uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5505 for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5506 {
5507 uint64_t u64Val;
5508 uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5509 VMXReadVmcs64(uVmcsField, &u64Val);
5510 IEMWriteVmxVmcsField(pVmcsNstGst, uVmcsField, u64Val);
5511 }
5512
5513 rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5514 rc |= hmR0VmxLoadVmcs(pVmcsInfo);
5515 }
5516 return rc;
5517}
5518
5519
5520/**
5521 * Enables VMCS shadowing for the given VMCS info. object.
5522 *
5523 * @param pVCpu The cross context virtual CPU structure.
5524 * @param pVmcsInfo The VMCS info. object.
5525 *
5526 * @remarks No-long-jump zone!!!
5527 */
5528static void hmR0VmxEnableVmcsShadowing(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5529{
5530 NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
5531
5532 uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5533 if (!(uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING))
5534 {
5535 Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
5536 uProcCtls2 |= VMX_PROC_CTLS2_VMCS_SHADOWING;
5537 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2);
5538 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, pVmcsInfo->HCPhysShadowVmcs);
5539 AssertRC(rc);
5540
5541 pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5542 pVmcsInfo->u64VmcsLinkPtr = pVmcsInfo->HCPhysShadowVmcs;
5543 Log4Func(("Enabled\n"));
5544 }
5545}
5546
5547
5548/**
5549 * Disables VMCS shadowing for the given VMCS info. object.
5550 *
5551 * @param pVCpu The cross context virtual CPU structure.
5552 * @param pVmcsInfo The VMCS info. object.
5553 *
5554 * @remarks No-long-jump zone!!!
5555 */
5556static void hmR0VmxDisableVmcsShadowing(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5557{
5558 NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
5559
5560 /*
5561 * We want all VMREAD and VMWRITE instructions to cause VM-exits, so we clear the
5562 * VMCS shadowing control. However, VM-entry requires the shadow VMCS indicator bit
5563 * to match the VMCS shadowing control if the VMCS link pointer is not NIL_RTHCPHYS.
5564 * Hence, we must also reset the VMCS link pointer to ensure VM-entry does not fail.
5565 *
5566 * See Intel spec. 26.2.1.1 "VM-Execution Control Fields".
5567 * See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5568 */
5569 uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5570 if (uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
5571 {
5572 uProcCtls2 &= ~VMX_PROC_CTLS2_VMCS_SHADOWING;
5573 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2);
5574 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
5575 AssertRC(rc);
5576
5577 pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5578 pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
5579 Log4Func(("Disabled\n"));
5580 }
5581}
5582#endif
5583
5584
5585/**
5586 * Exports the guest hardware-virtualization state.
5587 *
5588 * @returns VBox status code.
5589 * @param pVCpu The cross context virtual CPU structure.
5590 * @param pVmxTransient The VMX-transient structure.
5591 *
5592 * @remarks No-long-jump zone!!!
5593 */
5594static int hmR0VmxExportGuestHwvirtState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5595{
5596 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_HWVIRT)
5597 {
5598#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5599 /*
5600 * Check if the VMX feature is exposed to the guest and if the host CPU supports
5601 * VMCS shadowing.
5602 */
5603 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
5604 {
5605 /*
5606 * If the guest hypervisor has loaded a current VMCS and is in VMX root mode,
5607 * copy the guest hypervisor's current VMCS into the shadow VMCS and enable
5608 * VMCS shadowing to skip intercepting some or all VMREAD/VMWRITE VM-exits.
5609 *
5610 * We check for VMX root mode here in case the guest executes VMXOFF without
5611 * clearing the current VMCS pointer and our VMXOFF instruction emulation does
5612 * not clear the current VMCS pointer.
5613 */
5614 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5615 if ( CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx)
5616 && !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)
5617 && CPUMIsGuestVmxCurrentVmcsValid(pVCpu, &pVCpu->cpum.GstCtx))
5618 {
5619 /* Paranoia. */
5620 Assert(!pVmxTransient->fIsNestedGuest);
5621
5622 /*
5623 * For performance reasons, also check if the guest hypervisor's current VMCS
5624 * was newly loaded or modified before copying it to the shadow VMCS.
5625 */
5626 if (!pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs)
5627 {
5628 int rc = hmR0VmxCopyNstGstToShadowVmcs(pVCpu, pVmcsInfo);
5629 AssertRCReturn(rc, rc);
5630 pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs = true;
5631 }
5632 hmR0VmxEnableVmcsShadowing(pVCpu, pVmcsInfo);
5633 }
5634 else
5635 hmR0VmxDisableVmcsShadowing(pVCpu, pVmcsInfo);
5636 }
5637#else
5638 NOREF(pVmxTransient);
5639#endif
5640 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_HWVIRT);
5641 }
5642 return VINF_SUCCESS;
5643}
5644
5645
5646/**
5647 * Exports the guest CR0 control register into the guest-state area in the VMCS.
5648 *
5649 * The guest FPU state is always pre-loaded hence we don't need to bother about
5650 * sharing FPU related CR0 bits between the guest and host.
5651 *
5652 * @returns VBox status code.
5653 * @param pVCpu The cross context virtual CPU structure.
5654 * @param pVmxTransient The VMX-transient structure.
5655 *
5656 * @remarks No-long-jump zone!!!
5657 */
5658static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5659{
5660 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
5661 {
5662 PVM pVM = pVCpu->CTX_SUFF(pVM);
5663 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5664
5665 /*
5666 * Figure out fixed CR0 bits in VMX operation.
5667 */
5668 /** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
5669 uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
5670 uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
5671 if (pVM->hm.s.vmx.fUnrestrictedGuest)
5672 fSetCr0 &= ~(uint64_t)(X86_CR0_PE | X86_CR0_PG);
5673 else
5674 Assert((fSetCr0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
5675
5676 if (!pVmxTransient->fIsNestedGuest)
5677 {
5678 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5679 uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5680 uint64_t const u64ShadowCr0 = u64GuestCr0;
5681 Assert(!RT_HI_U32(u64GuestCr0));
5682
5683 /*
5684 * Setup VT-x's view of the guest CR0.
5685 */
5686 uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5687 if (pVM->hm.s.fNestedPaging)
5688 {
5689 if (CPUMIsGuestPagingEnabled(pVCpu))
5690 {
5691 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
5692 uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
5693 | VMX_PROC_CTLS_CR3_STORE_EXIT);
5694 }
5695 else
5696 {
5697 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
5698 uProcCtls |= VMX_PROC_CTLS_CR3_LOAD_EXIT
5699 | VMX_PROC_CTLS_CR3_STORE_EXIT;
5700 }
5701
5702 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
5703 if (pVM->hm.s.vmx.fUnrestrictedGuest)
5704 uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
5705 }
5706 else
5707 {
5708 /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
5709 u64GuestCr0 |= X86_CR0_WP;
5710 }
5711
5712 /*
5713 * Guest FPU bits.
5714 *
5715 * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
5716 * using CR0.TS.
5717 *
5718 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
5719 * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
5720 */
5721 u64GuestCr0 |= X86_CR0_NE;
5722
5723 /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
5724 bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
5725
5726 /*
5727 * Update exception intercepts.
5728 */
5729 uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
5730 if (pVmcsInfo->RealMode.fRealOnV86Active)
5731 {
5732 Assert(PDMVmmDevHeapIsEnabled(pVM));
5733 Assert(pVM->hm.s.vmx.pRealModeTSS);
5734 uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
5735 }
5736 else
5737 {
5738 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
5739 uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
5740 if (fInterceptMF)
5741 uXcptBitmap |= RT_BIT(X86_XCPT_MF);
5742 }
5743
5744 /* Additional intercepts for debugging, define these yourself explicitly. */
5745#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
5746 uXcptBitmap |= 0
5747 | RT_BIT(X86_XCPT_BP)
5748 | RT_BIT(X86_XCPT_DE)
5749 | RT_BIT(X86_XCPT_NM)
5750 | RT_BIT(X86_XCPT_TS)
5751 | RT_BIT(X86_XCPT_UD)
5752 | RT_BIT(X86_XCPT_NP)
5753 | RT_BIT(X86_XCPT_SS)
5754 | RT_BIT(X86_XCPT_GP)
5755 | RT_BIT(X86_XCPT_PF)
5756 | RT_BIT(X86_XCPT_MF)
5757 ;
5758#elif defined(HMVMX_ALWAYS_TRAP_PF)
5759 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
5760#endif
5761 if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
5762 uXcptBitmap |= RT_BIT(X86_XCPT_GP);
5763 Assert(pVM->hm.s.fNestedPaging || (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
5764
5765 /* Apply the hardware specified fixed CR0 bits and enable caching. */
5766 u64GuestCr0 |= fSetCr0;
5767 u64GuestCr0 &= fZapCr0;
5768 u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD | X86_CR0_NW);
5769
5770 /* Commit the CR0 and related fields to the guest VMCS. */
5771 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0); /** @todo Fix to 64-bit when we drop 32-bit. */
5772 rc |= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
5773 if (uProcCtls != pVmcsInfo->u32ProcCtls)
5774 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5775 if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
5776 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
5777 AssertRCReturn(rc, rc);
5778
5779 /* Update our caches. */
5780 pVmcsInfo->u32ProcCtls = uProcCtls;
5781 pVmcsInfo->u32XcptBitmap = uXcptBitmap;
5782
5783 Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5784 }
5785 else
5786 {
5787 /*
5788 * With nested-guests, we may have extended the guest/host mask here since we
5789 * merged in the outer guest's mask. Thus, the merged mask can include more bits
5790 * (to read from the nested-guest CR0 read-shadow) than the guest hypervisor
5791 * originally supplied. We must copy those bits from the nested-guest CR0 into
5792 * the nested-guest CR0 read-shadow.
5793 */
5794 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5795 uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5796 uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(pVCpu, &pVCpu->cpum.GstCtx, pVmcsInfo->u64Cr0Mask);
5797 Assert(!RT_HI_U32(u64GuestCr0));
5798 Assert(u64GuestCr0 & X86_CR0_NE);
5799
5800 /*
5801 * Apply the hardware specified fixed CR0 bits and enable caching.
5802 * Note! We could be altering our VMX emulation's fixed bits. We thus
5803 * need to re-apply them while importing CR0.
5804 */
5805 u64GuestCr0 |= fSetCr0;
5806 u64GuestCr0 &= fZapCr0;
5807 u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD | X86_CR0_NW);
5808
5809 /* Commit the CR0 and CR0 read-shadow to the nested-guest VMCS. */
5810 /** @todo NSTVMX: Fix to 64-bit when we drop 32-bit. */
5811 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0);
5812 rc |= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
5813 AssertRCReturn(rc, rc);
5814
5815 Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5816 }
5817
5818 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
5819 }
5820
5821 return VINF_SUCCESS;
5822}
5823
5824
5825/**
5826 * Exports the guest control registers (CR3, CR4) into the guest-state area
5827 * in the VMCS.
5828 *
5829 * @returns VBox strict status code.
5830 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
5831 * without unrestricted guest access and the VMMDev is not presently
5832 * mapped (e.g. EFI32).
5833 *
5834 * @param pVCpu The cross context virtual CPU structure.
5835 * @param pVmxTransient The VMX-transient structure.
5836 *
5837 * @remarks No-long-jump zone!!!
5838 */
5839static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5840{
5841 int rc = VINF_SUCCESS;
5842 PVM pVM = pVCpu->CTX_SUFF(pVM);
5843
5844 /*
5845 * Guest CR2.
5846 * It's always loaded in the assembler code. Nothing to do here.
5847 */
5848
5849 /*
5850 * Guest CR3.
5851 */
5852 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
5853 {
5854 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
5855
5856 RTGCPHYS GCPhysGuestCr3 = NIL_RTGCPHYS;
5857 if (pVM->hm.s.fNestedPaging)
5858 {
5859 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5860 pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
5861
5862 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
5863 Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
5864 Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
5865 Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
5866
5867 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
5868 pVmcsInfo->HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
5869 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
5870
5871 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
5872 AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
5873 && ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
5874 ("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
5875 AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
5876 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
5877 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
5878
5879 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
5880 AssertRCReturn(rc, rc);
5881
5882 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5883 if ( pVM->hm.s.vmx.fUnrestrictedGuest
5884 || CPUMIsGuestPagingEnabledEx(pCtx))
5885 {
5886 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
5887 if (CPUMIsGuestInPAEModeEx(pCtx))
5888 {
5889 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5890 AssertRCReturn(rc, rc);
5891 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
5892 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
5893 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
5894 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
5895 AssertRCReturn(rc, rc);
5896 }
5897
5898 /*
5899 * The guest's view of its CR3 is unblemished with nested paging when the
5900 * guest is using paging or we have unrestricted guest execution to handle
5901 * the guest when it's not using paging.
5902 */
5903 GCPhysGuestCr3 = pCtx->cr3;
5904 }
5905 else
5906 {
5907 /*
5908 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
5909 * thinks it accesses physical memory directly, we use our identity-mapped
5910 * page table to map guest-linear to guest-physical addresses. EPT takes care
5911 * of translating it to host-physical addresses.
5912 */
5913 RTGCPHYS GCPhys;
5914 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
5915
5916 /* We obtain it here every time as the guest could have relocated this PCI region. */
5917 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
5918 if (RT_SUCCESS(rc))
5919 { /* likely */ }
5920 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
5921 {
5922 Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
5923 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
5924 }
5925 else
5926 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
5927
5928 GCPhysGuestCr3 = GCPhys;
5929 }
5930
5931 Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCr3));
5932 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCr3);
5933 AssertRCReturn(rc, rc);
5934 }
5935 else
5936 {
5937 /* Non-nested paging case, just use the hypervisor's CR3. */
5938 RTHCPHYS const HCPhysGuestCr3 = PGMGetHyperCR3(pVCpu);
5939
5940 Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCr3));
5941 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCr3);
5942 AssertRCReturn(rc, rc);
5943 }
5944
5945 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
5946 }
5947
5948 /*
5949 * Guest CR4.
5950 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
5951 */
5952 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
5953 {
5954 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5955 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5956
5957 /*
5958 * Figure out fixed CR4 bits in VMX operation.
5959 */
5960 /** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
5961 uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5962 uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5963
5964 /*
5965 * With nested-guests, we may have extended the guest/host mask here (since we
5966 * merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
5967 * mask can include more bits (to read from the nested-guest CR4 read-shadow) than
5968 * the guest hypervisor originally supplied. Thus, we should, in essence, copy
5969 * those bits from the nested-guest CR4 into the nested-guest CR4 read-shadow.
5970 */
5971 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
5972 uint64_t u64GuestCr4 = pCtx->cr4;
5973 uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
5974 ? pCtx->cr4
5975 : CPUMGetGuestVmxMaskedCr4(pVCpu, pCtx, pVmcsInfo->u64Cr4Mask);
5976 Assert(!RT_HI_U32(u64GuestCr4));
5977
5978 /*
5979 * Setup VT-x's view of the guest CR4.
5980 *
5981 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
5982 * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
5983 * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
5984 *
5985 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5986 */
5987 if (pVmcsInfo->RealMode.fRealOnV86Active)
5988 {
5989 Assert(pVM->hm.s.vmx.pRealModeTSS);
5990 Assert(PDMVmmDevHeapIsEnabled(pVM));
5991 u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5992 }
5993
5994 if (pVM->hm.s.fNestedPaging)
5995 {
5996 if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5997 && !pVM->hm.s.vmx.fUnrestrictedGuest)
5998 {
5999 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
6000 u64GuestCr4 |= X86_CR4_PSE;
6001 /* Our identity mapping is a 32-bit page directory. */
6002 u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
6003 }
6004 /* else use guest CR4.*/
6005 }
6006 else
6007 {
6008 Assert(!pVmxTransient->fIsNestedGuest);
6009
6010 /*
6011 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
6012 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
6013 */
6014 switch (pVCpu->hm.s.enmShadowMode)
6015 {
6016 case PGMMODE_REAL: /* Real-mode. */
6017 case PGMMODE_PROTECTED: /* Protected mode without paging. */
6018 case PGMMODE_32_BIT: /* 32-bit paging. */
6019 {
6020 u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
6021 break;
6022 }
6023
6024 case PGMMODE_PAE: /* PAE paging. */
6025 case PGMMODE_PAE_NX: /* PAE paging with NX. */
6026 {
6027 u64GuestCr4 |= X86_CR4_PAE;
6028 break;
6029 }
6030
6031 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
6032 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
6033#ifdef VBOX_ENABLE_64_BITS_GUESTS
6034 break;
6035#endif
6036 default:
6037 AssertFailed();
6038 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
6039 }
6040 }
6041
6042 /*
6043 * Apply the hardware specified fixed CR4 bits (mainly CR4.VMXE).
6044 * Note! For nested-guests, we could be altering our VMX emulation's
6045 * fixed bits. We thus need to re-apply them while importing CR4.
6046 */
6047 u64GuestCr4 |= fSetCr4;
6048 u64GuestCr4 &= fZapCr4;
6049
6050 /* Commit the CR4 and CR4 read-shadow to the guest VMCS. */
6051 /** @todo Fix to 64-bit when we drop 32-bit. */
6052 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u64GuestCr4);
6053 rc |= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);
6054 AssertRCReturn(rc, rc);
6055
6056 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
6057 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
6058
6059 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
6060
6061 Log4Func(("cr4=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
6062 }
6063 return rc;
6064}
6065
6066
6067/**
6068 * Exports the guest debug registers into the guest-state area in the VMCS.
6069 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
6070 *
6071 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
6072 *
6073 * @returns VBox status code.
6074 * @param pVCpu The cross context virtual CPU structure.
6075 * @param pVmxTransient The VMX-transient structure.
6076 *
6077 * @remarks No-long-jump zone!!!
6078 */
6079static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6080{
6081 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6082
6083 /** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
6084 * stepping. */
6085 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6086 if (pVmxTransient->fIsNestedGuest)
6087 {
6088 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
6089 AssertRCReturn(rc, rc);
6090 return VINF_SUCCESS;
6091 }
6092
6093#ifdef VBOX_STRICT
6094 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
6095 if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
6096 {
6097 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
6098 Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0);
6099 Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
6100 }
6101#endif
6102
6103 bool fSteppingDB = false;
6104 bool fInterceptMovDRx = false;
6105 uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
6106 if (pVCpu->hm.s.fSingleInstruction)
6107 {
6108 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
6109 PVM pVM = pVCpu->CTX_SUFF(pVM);
6110 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
6111 {
6112 uProcCtls |= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
6113 Assert(fSteppingDB == false);
6114 }
6115 else
6116 {
6117 pVCpu->cpum.GstCtx.eflags.u32 |= X86_EFL_TF;
6118 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
6119 pVCpu->hm.s.fClearTrapFlag = true;
6120 fSteppingDB = true;
6121 }
6122 }
6123
6124 uint32_t u32GuestDr7;
6125 if ( fSteppingDB
6126 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
6127 {
6128 /*
6129 * Use the combined guest and host DRx values found in the hypervisor register set
6130 * because the hypervisor debugger has breakpoints active or someone is single stepping
6131 * on the host side without a monitor trap flag.
6132 *
6133 * Note! DBGF expects a clean DR6 state before executing guest code.
6134 */
6135#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
6136 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6137 && !CPUMIsHyperDebugStateActivePending(pVCpu))
6138 {
6139 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
6140 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
6141 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
6142 }
6143 else
6144#endif
6145 if (!CPUMIsHyperDebugStateActive(pVCpu))
6146 {
6147 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
6148 Assert(CPUMIsHyperDebugStateActive(pVCpu));
6149 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
6150 }
6151
6152 /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
6153 u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
6154 pVCpu->hm.s.fUsingHyperDR7 = true;
6155 fInterceptMovDRx = true;
6156 }
6157 else
6158 {
6159 /*
6160 * If the guest has enabled debug registers, we need to load them prior to
6161 * executing guest code so they'll trigger at the right time.
6162 */
6163 if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD))
6164 {
6165#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
6166 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6167 && !CPUMIsGuestDebugStateActivePending(pVCpu))
6168 {
6169 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
6170 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
6171 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
6172 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
6173 }
6174 else
6175#endif
6176 if (!CPUMIsGuestDebugStateActive(pVCpu))
6177 {
6178 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
6179 Assert(CPUMIsGuestDebugStateActive(pVCpu));
6180 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
6181 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
6182 }
6183 Assert(!fInterceptMovDRx);
6184 }
6185 /*
6186 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
6187 * must intercept #DB in order to maintain a correct DR6 guest value, and
6188 * because we need to intercept it to prevent nested #DBs from hanging the
6189 * CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
6190 */
6191#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
6192 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
6193 && !CPUMIsGuestDebugStateActive(pVCpu))
6194#else
6195 else if (!CPUMIsGuestDebugStateActive(pVCpu))
6196#endif
6197 {
6198 fInterceptMovDRx = true;
6199 }
6200
6201 /* Update DR7 with the actual guest value. */
6202 u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
6203 pVCpu->hm.s.fUsingHyperDR7 = false;
6204 }
6205
6206 if (fInterceptMovDRx)
6207 uProcCtls |= VMX_PROC_CTLS_MOV_DR_EXIT;
6208 else
6209 uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
6210
6211 /*
6212 * Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
6213 * monitor-trap flag and update our cache.
6214 */
6215 if (uProcCtls != pVmcsInfo->u32ProcCtls)
6216 {
6217 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6218 AssertRCReturn(rc2, rc2);
6219 pVmcsInfo->u32ProcCtls = uProcCtls;
6220 }
6221
6222 /*
6223 * Update guest DR7.
6224 */
6225 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
6226 AssertRCReturn(rc, rc);
6227
6228 /*
6229 * If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
6230 * we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
6231 *
6232 * See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
6233 */
6234 if (fSteppingDB)
6235 {
6236 Assert(pVCpu->hm.s.fSingleInstruction);
6237 Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
6238
6239 uint32_t fIntrState = 0;
6240 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
6241 AssertRCReturn(rc, rc);
6242
6243 if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
6244 {
6245 fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
6246 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
6247 AssertRCReturn(rc, rc);
6248 }
6249 }
6250
6251 return VINF_SUCCESS;
6252}
6253
6254
6255#ifdef VBOX_STRICT
6256/**
6257 * Strict function to validate segment registers.
6258 *
6259 * @param pVCpu The cross context virtual CPU structure.
6260 * @param pVmcsInfo The VMCS info. object.
6261 *
6262 * @remarks Will import guest CR0 on strict builds during validation of
6263 * segments.
6264 */
6265static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
6266{
6267 /*
6268 * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6269 *
6270 * The reason we check for attribute value 0 in this function and not just the unusable bit is
6271 * because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
6272 * unusable bit and doesn't change the guest-context value.
6273 */
6274 PVM pVM = pVCpu->CTX_SUFF(pVM);
6275 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6276 hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
6277 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6278 && ( !CPUMIsGuestInRealModeEx(pCtx)
6279 && !CPUMIsGuestInV86ModeEx(pCtx)))
6280 {
6281 /* Protected mode checks */
6282 /* CS */
6283 Assert(pCtx->cs.Attr.n.u1Present);
6284 Assert(!(pCtx->cs.Attr.u & 0xf00));
6285 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
6286 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
6287 || !(pCtx->cs.Attr.n.u1Granularity));
6288 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
6289 || (pCtx->cs.Attr.n.u1Granularity));
6290 /* CS cannot be loaded with NULL in protected mode. */
6291 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
6292 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
6293 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
6294 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
6295 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
6296 else
6297 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
6298 /* SS */
6299 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
6300 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
6301 if ( !(pCtx->cr0 & X86_CR0_PE)
6302 || pCtx->cs.Attr.n.u4Type == 3)
6303 {
6304 Assert(!pCtx->ss.Attr.n.u2Dpl);
6305 }
6306 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
6307 {
6308 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
6309 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
6310 Assert(pCtx->ss.Attr.n.u1Present);
6311 Assert(!(pCtx->ss.Attr.u & 0xf00));
6312 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
6313 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
6314 || !(pCtx->ss.Attr.n.u1Granularity));
6315 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
6316 || (pCtx->ss.Attr.n.u1Granularity));
6317 }
6318 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
6319 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
6320 {
6321 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6322 Assert(pCtx->ds.Attr.n.u1Present);
6323 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
6324 Assert(!(pCtx->ds.Attr.u & 0xf00));
6325 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
6326 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
6327 || !(pCtx->ds.Attr.n.u1Granularity));
6328 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
6329 || (pCtx->ds.Attr.n.u1Granularity));
6330 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6331 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
6332 }
6333 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
6334 {
6335 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6336 Assert(pCtx->es.Attr.n.u1Present);
6337 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
6338 Assert(!(pCtx->es.Attr.u & 0xf00));
6339 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
6340 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
6341 || !(pCtx->es.Attr.n.u1Granularity));
6342 Assert( !(pCtx->es.u32Limit & 0xfff00000)
6343 || (pCtx->es.Attr.n.u1Granularity));
6344 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6345 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
6346 }
6347 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
6348 {
6349 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6350 Assert(pCtx->fs.Attr.n.u1Present);
6351 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
6352 Assert(!(pCtx->fs.Attr.u & 0xf00));
6353 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
6354 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
6355 || !(pCtx->fs.Attr.n.u1Granularity));
6356 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
6357 || (pCtx->fs.Attr.n.u1Granularity));
6358 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6359 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6360 }
6361 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
6362 {
6363 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6364 Assert(pCtx->gs.Attr.n.u1Present);
6365 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
6366 Assert(!(pCtx->gs.Attr.u & 0xf00));
6367 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
6368 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
6369 || !(pCtx->gs.Attr.n.u1Granularity));
6370 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
6371 || (pCtx->gs.Attr.n.u1Granularity));
6372 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6373 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6374 }
6375 /* 64-bit capable CPUs. */
6376# if HC_ARCH_BITS == 64
6377 Assert(!RT_HI_U32(pCtx->cs.u64Base));
6378 Assert(!pCtx->ss.Attr.u || !RT_HI_U32(pCtx->ss.u64Base));
6379 Assert(!pCtx->ds.Attr.u || !RT_HI_U32(pCtx->ds.u64Base));
6380 Assert(!pCtx->es.Attr.u || !RT_HI_U32(pCtx->es.u64Base));
6381# endif
6382 }
6383 else if ( CPUMIsGuestInV86ModeEx(pCtx)
6384 || ( CPUMIsGuestInRealModeEx(pCtx)
6385 && !pVM->hm.s.vmx.fUnrestrictedGuest))
6386 {
6387 /* Real and v86 mode checks. */
6388 /* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
6389 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
6390 if (pVmcsInfo->RealMode.fRealOnV86Active)
6391 {
6392 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
6393 u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
6394 }
6395 else
6396 {
6397 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
6398 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
6399 }
6400
6401 /* CS */
6402 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
6403 Assert(pCtx->cs.u32Limit == 0xffff);
6404 Assert(u32CSAttr == 0xf3);
6405 /* SS */
6406 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
6407 Assert(pCtx->ss.u32Limit == 0xffff);
6408 Assert(u32SSAttr == 0xf3);
6409 /* DS */
6410 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
6411 Assert(pCtx->ds.u32Limit == 0xffff);
6412 Assert(u32DSAttr == 0xf3);
6413 /* ES */
6414 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
6415 Assert(pCtx->es.u32Limit == 0xffff);
6416 Assert(u32ESAttr == 0xf3);
6417 /* FS */
6418 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
6419 Assert(pCtx->fs.u32Limit == 0xffff);
6420 Assert(u32FSAttr == 0xf3);
6421 /* GS */
6422 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
6423 Assert(pCtx->gs.u32Limit == 0xffff);
6424 Assert(u32GSAttr == 0xf3);
6425 /* 64-bit capable CPUs. */
6426# if HC_ARCH_BITS == 64
6427 Assert(!RT_HI_U32(pCtx->cs.u64Base));
6428 Assert(!u32SSAttr || !RT_HI_U32(pCtx->ss.u64Base));
6429 Assert(!u32DSAttr || !RT_HI_U32(pCtx->ds.u64Base));
6430 Assert(!u32ESAttr || !RT_HI_U32(pCtx->es.u64Base));
6431# endif
6432 }
6433}
6434#endif /* VBOX_STRICT */
6435
6436
6437/**
6438 * Exports a guest segment register into the guest-state area in the VMCS.
6439 *
6440 * @returns VBox status code.
6441 * @param pVCpu The cross context virtual CPU structure.
6442 * @param pVmcsInfo The VMCS info. object.
6443 * @param iSegReg The segment register number (X86_SREG_XXX).
6444 * @param pSelReg Pointer to the segment selector.
6445 *
6446 * @remarks No-long-jump zone!!!
6447 */
6448static int hmR0VmxExportGuestSegReg(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
6449{
6450 Assert(iSegReg < X86_SREG_COUNT);
6451 uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
6452 uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
6453 uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
6454 uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
6455
6456 uint32_t u32Access = pSelReg->Attr.u;
6457 if (pVmcsInfo->RealMode.fRealOnV86Active)
6458 {
6459 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
6460 u32Access = 0xf3;
6461 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6462 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
6463 RT_NOREF_PV(pVCpu);
6464 }
6465 else
6466 {
6467 /*
6468 * The way to differentiate between whether this is really a null selector or was just
6469 * a selector loaded with 0 in real-mode is using the segment attributes. A selector
6470 * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
6471 * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
6472 * NULL selectors loaded in protected-mode have their attribute as 0.
6473 */
6474 if (!u32Access)
6475 u32Access = X86DESCATTR_UNUSABLE;
6476 }
6477
6478 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
6479 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
6480 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
6481
6482 /*
6483 * Commit it to the VMCS.
6484 */
6485 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel);
6486 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit);
6487 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base);
6488 rc |= VMXWriteVmcs32(idxAttr, u32Access);
6489 AssertRCReturn(rc, rc);
6490 return rc;
6491}
6492
6493
6494/**
6495 * Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
6496 * area in the VMCS.
6497 *
6498 * @returns VBox status code.
6499 * @param pVCpu The cross context virtual CPU structure.
6500 * @param pVmxTransient The VMX-transient structure.
6501 *
6502 * @remarks Will import guest CR0 on strict builds during validation of
6503 * segments.
6504 * @remarks No-long-jump zone!!!
6505 */
6506static int hmR0VmxExportGuestSegRegsXdtr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6507{
6508 int rc = VERR_INTERNAL_ERROR_5;
6509 PVM pVM = pVCpu->CTX_SUFF(pVM);
6510 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6511 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6512
6513 /*
6514 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
6515 */
6516 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
6517 {
6518#ifdef VBOX_WITH_REM
6519 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6520 {
6521 Assert(!pVmxTransient->fIsNestedGuest);
6522 Assert(pVM->hm.s.vmx.pRealModeTSS);
6523 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
6524 if ( pVmcsInfo->fWasInRealMode
6525 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
6526 {
6527 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
6528 in real-mode (e.g. OpenBSD 4.0) */
6529 REMFlushTBs(pVM);
6530 Log4Func(("Switch to protected mode detected!\n"));
6531 pVmcsInfo->fWasInRealMode = false;
6532 }
6533 }
6534#endif
6535 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
6536 {
6537 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
6538 if (pVmcsInfo->RealMode.fRealOnV86Active)
6539 pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
6540 rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
6541 AssertRCReturn(rc, rc);
6542 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
6543 }
6544
6545 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
6546 {
6547 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
6548 if (pVmcsInfo->RealMode.fRealOnV86Active)
6549 pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
6550 rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
6551 AssertRCReturn(rc, rc);
6552 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
6553 }
6554
6555 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
6556 {
6557 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
6558 if (pVmcsInfo->RealMode.fRealOnV86Active)
6559 pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
6560 rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
6561 AssertRCReturn(rc, rc);
6562 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
6563 }
6564
6565 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
6566 {
6567 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
6568 if (pVmcsInfo->RealMode.fRealOnV86Active)
6569 pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
6570 rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
6571 AssertRCReturn(rc, rc);
6572 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
6573 }
6574
6575 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
6576 {
6577 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
6578 if (pVmcsInfo->RealMode.fRealOnV86Active)
6579 pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
6580 rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
6581 AssertRCReturn(rc, rc);
6582 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
6583 }
6584
6585 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
6586 {
6587 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
6588 if (pVmcsInfo->RealMode.fRealOnV86Active)
6589 pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
6590 rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
6591 AssertRCReturn(rc, rc);
6592 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
6593 }
6594
6595#ifdef VBOX_STRICT
6596 hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
6597#endif
6598 Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
6599 pCtx->cs.Attr.u));
6600 }
6601
6602 /*
6603 * Guest TR.
6604 */
6605 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
6606 {
6607 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
6608
6609 /*
6610 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
6611 * achieved using the interrupt redirection bitmap (all bits cleared to let the guest
6612 * handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
6613 */
6614 uint16_t u16Sel;
6615 uint32_t u32Limit;
6616 uint64_t u64Base;
6617 uint32_t u32AccessRights;
6618 if (!pVmcsInfo->RealMode.fRealOnV86Active)
6619 {
6620 u16Sel = pCtx->tr.Sel;
6621 u32Limit = pCtx->tr.u32Limit;
6622 u64Base = pCtx->tr.u64Base;
6623 u32AccessRights = pCtx->tr.Attr.u;
6624 }
6625 else
6626 {
6627 Assert(!pVmxTransient->fIsNestedGuest);
6628 Assert(pVM->hm.s.vmx.pRealModeTSS);
6629 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
6630
6631 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
6632 RTGCPHYS GCPhys;
6633 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
6634 AssertRCReturn(rc, rc);
6635
6636 X86DESCATTR DescAttr;
6637 DescAttr.u = 0;
6638 DescAttr.n.u1Present = 1;
6639 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
6640
6641 u16Sel = 0;
6642 u32Limit = HM_VTX_TSS_SIZE;
6643 u64Base = GCPhys;
6644 u32AccessRights = DescAttr.u;
6645 }
6646
6647 /* Validate. */
6648 Assert(!(u16Sel & RT_BIT(2)));
6649 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
6650 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
6651 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
6652 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
6653 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
6654 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
6655 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
6656 Assert( (u32Limit & 0xfff) == 0xfff
6657 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
6658 Assert( !(pCtx->tr.u32Limit & 0xfff00000)
6659 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
6660
6661 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
6662 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
6663 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
6664 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
6665 AssertRCReturn(rc, rc);
6666
6667 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
6668 Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
6669 }
6670
6671 /*
6672 * Guest GDTR.
6673 */
6674 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
6675 {
6676 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
6677
6678 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
6679 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
6680 AssertRCReturn(rc, rc);
6681
6682 /* Validate. */
6683 Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6684
6685 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
6686 Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
6687 }
6688
6689 /*
6690 * Guest LDTR.
6691 */
6692 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
6693 {
6694 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
6695
6696 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
6697 uint32_t u32Access;
6698 if ( !pVmxTransient->fIsNestedGuest
6699 && !pCtx->ldtr.Attr.u)
6700 u32Access = X86DESCATTR_UNUSABLE;
6701 else
6702 u32Access = pCtx->ldtr.Attr.u;
6703
6704 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
6705 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
6706 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
6707 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
6708 AssertRCReturn(rc, rc);
6709
6710 /* Validate. */
6711 if (!(u32Access & X86DESCATTR_UNUSABLE))
6712 {
6713 Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
6714 Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
6715 Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
6716 Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
6717 Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
6718 Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
6719 Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
6720 || !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
6721 Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
6722 || pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
6723 }
6724
6725 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
6726 Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
6727 }
6728
6729 /*
6730 * Guest IDTR.
6731 */
6732 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
6733 {
6734 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
6735
6736 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
6737 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
6738 AssertRCReturn(rc, rc);
6739
6740 /* Validate. */
6741 Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6742
6743 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
6744 Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
6745 }
6746
6747 return VINF_SUCCESS;
6748}
6749
6750
6751/**
6752 * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
6753 * areas.
6754 *
6755 * These MSRs will automatically be loaded to the host CPU on every successful
6756 * VM-entry and stored from the host CPU on every successful VM-exit.
6757 *
6758 * We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
6759 * actual host MSR values are not- updated here for performance reasons. See
6760 * hmR0VmxExportHostMsrs().
6761 *
6762 * We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
6763 *
6764 * @returns VBox status code.
6765 * @param pVCpu The cross context virtual CPU structure.
6766 * @param pVmxTransient The VMX-transient structure.
6767 *
6768 * @remarks No-long-jump zone!!!
6769 */
6770static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6771{
6772 AssertPtr(pVCpu);
6773 AssertPtr(pVmxTransient);
6774
6775 PVM pVM = pVCpu->CTX_SUFF(pVM);
6776 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6777
6778 /*
6779 * MSRs that we use the auto-load/store MSR area in the VMCS.
6780 * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
6781 * The host MSR values are updated when it's safe in hmR0VmxLazySaveHostMsrs().
6782 *
6783 * For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
6784 * loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
6785 * emulation, nothing to do here.
6786 */
6787 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
6788 {
6789 if ( !pVmxTransient->fIsNestedGuest
6790 && pVM->hm.s.fAllow64BitGuests)
6791 {
6792#if HC_ARCH_BITS == 32
6793 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS | CPUMCTX_EXTRN_KERNEL_GS_BASE);
6794 Assert(!pVmxTransient->fIsNestedGuest);
6795
6796 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_LSTAR, pCtx->msrLSTAR, true, false);
6797 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_STAR, pCtx->msrSTAR, true, false);
6798 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_SF_MASK, pCtx->msrSFMASK, true, false);
6799 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, true, false);
6800 AssertRCReturn(rc, rc);
6801#endif
6802 }
6803 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
6804 }
6805
6806 /*
6807 * Guest Sysenter MSRs.
6808 */
6809 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
6810 {
6811 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
6812
6813 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
6814 {
6815 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
6816 AssertRCReturn(rc, rc);
6817 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
6818 }
6819
6820 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
6821 {
6822 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
6823 AssertRCReturn(rc, rc);
6824 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
6825 }
6826
6827 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
6828 {
6829 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
6830 AssertRCReturn(rc, rc);
6831 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
6832 }
6833 }
6834
6835 /*
6836 * Guest/host EFER MSR.
6837 */
6838 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
6839 {
6840 /* Whether we are using the VMCS to swap the EFER MSR must have been
6841 determined earlier while exporting VM-entry/VM-exit controls. */
6842 Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
6843 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
6844
6845 if (hmR0VmxShouldSwapEferMsr(pVCpu))
6846 {
6847 /*
6848 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
6849 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
6850 */
6851 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
6852 {
6853 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
6854 AssertRCReturn(rc, rc);
6855 }
6856 else
6857 {
6858 /*
6859 * We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
6860 * continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
6861 */
6862 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, pCtx->msrEFER,
6863 false /* fSetReadWrite */, false /* fUpdateHostMsr */);
6864 AssertRCReturn(rc, rc);
6865 }
6866 }
6867 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
6868 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
6869
6870 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
6871 }
6872
6873 /*
6874 * Other MSRs.
6875 * Speculation Control (R/W).
6876 */
6877 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
6878 {
6879 HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
6880 if (pVM->cpum.ro.GuestFeatures.fIbrs)
6881 {
6882 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
6883 false /* fSetReadWrite */, false /* fUpdateHostMsr */);
6884 AssertRCReturn(rc, rc);
6885 }
6886 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
6887 }
6888
6889 return VINF_SUCCESS;
6890}
6891
6892
6893/**
6894 * Selects up the appropriate function to run guest code.
6895 *
6896 * @returns VBox status code.
6897 * @param pVCpu The cross context virtual CPU structure.
6898 * @param pVmxTransient The VMX-transient structure.
6899 *
6900 * @remarks No-long-jump zone!!!
6901 */
6902static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6903{
6904 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6905 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6906
6907 if (CPUMIsGuestInLongModeEx(pCtx))
6908 {
6909#ifndef VBOX_ENABLE_64_BITS_GUESTS
6910 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
6911#endif
6912 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
6913#if HC_ARCH_BITS == 32
6914 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
6915 if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
6916 {
6917#ifdef VBOX_STRICT
6918 if (pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
6919 {
6920 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
6921 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
6922 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
6923 AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS | HM_CHANGED_GUEST_EFER_MSR),
6924 ("fCtxChanged=%#RX64\n", fCtxChanged));
6925 }
6926#endif
6927 pVmcsInfo->pfnStartVM = VMXR0SwitcherStartVM64;
6928
6929 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
6930 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
6931 pVmcsInfo->fSwitchedTo64on32 = true;
6932 Log4Func(("Selected 64-bit switcher\n"));
6933 }
6934#else
6935 /* 64-bit host. */
6936 pVmcsInfo->pfnStartVM = VMXR0StartVM64;
6937#endif
6938 }
6939 else
6940 {
6941 /* Guest is not in long mode, use the 32-bit handler. */
6942#if HC_ARCH_BITS == 32
6943 if ( pVmcsInfo->pfnStartVM != VMXR0StartVM32
6944 && !pVmcsInfo->fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
6945 && pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
6946 {
6947# ifdef VBOX_STRICT
6948 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
6949 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
6950 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
6951 AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS | HM_CHANGED_GUEST_EFER_MSR),
6952 ("fCtxChanged=%#RX64\n", fCtxChanged));
6953# endif
6954 }
6955# ifdef VBOX_ENABLE_64_BITS_GUESTS
6956 /*
6957 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
6958 * design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
6959 * switcher flag now because we know the guest is in a sane state where it's safe
6960 * to use the 32-bit switcher. Otherwise, check the guest state if it's safe to use
6961 * the much faster 32-bit switcher again.
6962 */
6963 if (!pVmcsInfo->fSwitchedTo64on32)
6964 {
6965 if (pVmcsInfo->pfnStartVM != VMXR0StartVM32)
6966 Log4Func(("Selected 32-bit switcher\n"));
6967 pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6968 }
6969 else
6970 {
6971 Assert(pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64);
6972 if ( pVmcsInfo->RealMode.fRealOnV86Active
6973 || hmR0VmxIs32BitSwitcherSafe(pCtx))
6974 {
6975 pVmcsInfo->fSwitchedTo64on32 = false;
6976 pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6977 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
6978 | HM_CHANGED_VMX_ENTRY_EXIT_CTLS
6979 | HM_CHANGED_HOST_CONTEXT);
6980 Log4Func(("Selected 32-bit switcher (safe)\n"));
6981 }
6982 }
6983# else
6984 pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6985# endif
6986#else
6987 pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6988#endif
6989 }
6990 Assert(pVmcsInfo->pfnStartVM);
6991 return VINF_SUCCESS;
6992}
6993
6994
6995/**
6996 * Wrapper for running the guest code in VT-x.
6997 *
6998 * @returns VBox status code, no informational status codes.
6999 * @param pVCpu The cross context virtual CPU structure.
7000 * @param pVmxTransient The VMX-transient structure.
7001 *
7002 * @remarks No-long-jump zone!!!
7003 */
7004DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient)
7005{
7006 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
7007 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7008 pCtx->fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
7009
7010 /** @todo Add stats for VMRESUME vs VMLAUNCH. */
7011
7012 /*
7013 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
7014 * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
7015 * callee-saved and thus the need for this XMM wrapper.
7016 *
7017 * See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
7018 */
7019 PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7020 bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
7021 PVM pVM = pVCpu->CTX_SUFF(pVM);
7022#ifdef VBOX_WITH_KERNEL_USING_XMM
7023 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu, pVmcsInfo->pfnStartVM);
7024#else
7025 int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu);
7026#endif
7027 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
7028 return rc;
7029}
7030
7031
7032/**
7033 * Reports world-switch error and dumps some useful debug info.
7034 *
7035 * @param pVCpu The cross context virtual CPU structure.
7036 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
7037 * @param pVmxTransient The VMX-transient structure (only
7038 * exitReason updated).
7039 */
7040static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
7041{
7042 Assert(pVCpu);
7043 Assert(pVmxTransient);
7044 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7045
7046 Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
7047 switch (rcVMRun)
7048 {
7049 case VERR_VMX_INVALID_VMXON_PTR:
7050 AssertFailed();
7051 break;
7052 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
7053 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
7054 {
7055 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
7056 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
7057 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
7058 AssertRC(rc);
7059
7060 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7061 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
7062 Cannot do it here as we may have been long preempted. */
7063
7064#ifdef VBOX_STRICT
7065 PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7066 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
7067 pVmxTransient->uExitReason));
7068 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
7069 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
7070 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
7071 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
7072 else
7073 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
7074 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
7075 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
7076
7077 /* VMX control bits. */
7078 uint32_t u32Val;
7079 uint64_t u64Val;
7080 RTHCUINTREG uHCReg;
7081 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
7082 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
7083 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
7084 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
7085 if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
7086 {
7087 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
7088 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
7089 }
7090 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
7091 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
7092 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
7093 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
7094 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
7095 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
7096 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
7097 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
7098 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
7099 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
7100 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
7101 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
7102 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
7103 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
7104 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
7105 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
7106 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
7107 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
7108 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
7109 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
7110 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
7111 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
7112 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
7113 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
7114 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
7115 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
7116 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7117 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7118 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7119 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7120 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7121 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7122 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7123 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7124 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
7125 {
7126 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7127 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7128 }
7129
7130 /* Guest bits. */
7131 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
7132 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
7133 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
7134 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
7135 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
7136 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
7137 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
7138 {
7139 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
7140 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
7141 }
7142
7143 /* Host bits. */
7144 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
7145 Log4(("Host CR0 %#RHr\n", uHCReg));
7146 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
7147 Log4(("Host CR3 %#RHr\n", uHCReg));
7148 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
7149 Log4(("Host CR4 %#RHr\n", uHCReg));
7150
7151 RTGDTR HostGdtr;
7152 PCX86DESCHC pDesc;
7153 ASMGetGDTR(&HostGdtr);
7154 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
7155 Log4(("Host CS %#08x\n", u32Val));
7156 if (u32Val < HostGdtr.cbGdt)
7157 {
7158 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7159 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
7160 }
7161
7162 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
7163 Log4(("Host DS %#08x\n", u32Val));
7164 if (u32Val < HostGdtr.cbGdt)
7165 {
7166 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7167 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
7168 }
7169
7170 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
7171 Log4(("Host ES %#08x\n", u32Val));
7172 if (u32Val < HostGdtr.cbGdt)
7173 {
7174 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7175 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
7176 }
7177
7178 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
7179 Log4(("Host FS %#08x\n", u32Val));
7180 if (u32Val < HostGdtr.cbGdt)
7181 {
7182 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7183 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
7184 }
7185
7186 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
7187 Log4(("Host GS %#08x\n", u32Val));
7188 if (u32Val < HostGdtr.cbGdt)
7189 {
7190 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7191 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
7192 }
7193
7194 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
7195 Log4(("Host SS %#08x\n", u32Val));
7196 if (u32Val < HostGdtr.cbGdt)
7197 {
7198 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7199 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
7200 }
7201
7202 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
7203 Log4(("Host TR %#08x\n", u32Val));
7204 if (u32Val < HostGdtr.cbGdt)
7205 {
7206 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7207 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
7208 }
7209
7210 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
7211 Log4(("Host TR Base %#RHv\n", uHCReg));
7212 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
7213 Log4(("Host GDTR Base %#RHv\n", uHCReg));
7214 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
7215 Log4(("Host IDTR Base %#RHv\n", uHCReg));
7216 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
7217 Log4(("Host SYSENTER CS %#08x\n", u32Val));
7218 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
7219 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
7220 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
7221 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
7222 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
7223 Log4(("Host RSP %#RHv\n", uHCReg));
7224 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
7225 Log4(("Host RIP %#RHv\n", uHCReg));
7226# if HC_ARCH_BITS == 64
7227 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
7228 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
7229 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
7230 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
7231 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
7232 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
7233# endif
7234#endif /* VBOX_STRICT */
7235 break;
7236 }
7237
7238 default:
7239 /* Impossible */
7240 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
7241 break;
7242 }
7243}
7244
7245
7246#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
7247# ifndef VMX_USE_CACHED_VMCS_ACCESSES
7248# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
7249# endif
7250
7251/**
7252 * Initialize the VMCS-Read cache.
7253 *
7254 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
7255 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
7256 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
7257 * (those that have a 32-bit FULL & HIGH part).
7258 *
7259 * @param pVCpu The cross context virtual CPU structure.
7260 */
7261static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
7262{
7263#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
7264 do { \
7265 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
7266 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
7267 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
7268 ++cReadFields; \
7269 } while (0)
7270
7271 PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
7272 uint32_t cReadFields = 0;
7273
7274 /*
7275 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
7276 * and serve to indicate exceptions to the rules.
7277 */
7278
7279 /* Guest-natural selector base fields. */
7280#if 0
7281 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
7282 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
7283 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
7284#endif
7285 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
7286 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
7287 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
7288 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
7289 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
7290 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
7291 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
7292 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
7293 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
7294 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
7295 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
7296 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
7297#if 0
7298 /* Unused natural width guest-state fields. */
7299 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS);
7300 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in nested paging case */
7301#endif
7302 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
7303 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
7304
7305 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
7306 these 64-bit fields (using "FULL" and "HIGH" fields). */
7307#if 0
7308 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
7309 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
7310 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
7311 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
7312 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
7313 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
7314 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
7315 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
7316 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
7317#endif
7318
7319 /* Natural width guest-state fields. */
7320 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
7321 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_GUEST_LINEAR_ADDR);
7322
7323 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
7324 {
7325 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
7326 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
7327 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
7328 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
7329 }
7330 else
7331 {
7332 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
7333 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
7334 }
7335
7336#undef VMXLOCAL_INIT_READ_CACHE_FIELD
7337}
7338
7339
7340/**
7341 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
7342 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
7343 * darwin, running 64-bit guests).
7344 *
7345 * @returns VBox status code.
7346 * @param pVCpu The cross context virtual CPU structure.
7347 * @param idxField The VMCS field encoding.
7348 * @param u64Val 16, 32 or 64-bit value.
7349 */
7350VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
7351{
7352 AssertPtr(pVCpu);
7353 int rc;
7354 switch (idxField)
7355 {
7356 /*
7357 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
7358 */
7359 /* 64-bit Control fields. */
7360 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
7361 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
7362 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
7363 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
7364 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
7365 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
7366 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
7367 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
7368 case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
7369 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
7370 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
7371 case VMX_VMCS64_CTRL_EPTP_FULL:
7372 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
7373 /* 64-bit Guest-state fields. */
7374 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
7375 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
7376 case VMX_VMCS64_GUEST_PAT_FULL:
7377 case VMX_VMCS64_GUEST_EFER_FULL:
7378 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
7379 case VMX_VMCS64_GUEST_PDPTE0_FULL:
7380 case VMX_VMCS64_GUEST_PDPTE1_FULL:
7381 case VMX_VMCS64_GUEST_PDPTE2_FULL:
7382 case VMX_VMCS64_GUEST_PDPTE3_FULL:
7383 /* 64-bit Host-state fields. */
7384 case VMX_VMCS64_HOST_PAT_FULL:
7385 case VMX_VMCS64_HOST_EFER_FULL:
7386 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
7387 {
7388 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
7389 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
7390 break;
7391 }
7392
7393 /*
7394 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
7395 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
7396 */
7397 /* Natural-width Guest-state fields. */
7398 case VMX_VMCS_GUEST_CR3:
7399 case VMX_VMCS_GUEST_ES_BASE:
7400 case VMX_VMCS_GUEST_CS_BASE:
7401 case VMX_VMCS_GUEST_SS_BASE:
7402 case VMX_VMCS_GUEST_DS_BASE:
7403 case VMX_VMCS_GUEST_FS_BASE:
7404 case VMX_VMCS_GUEST_GS_BASE:
7405 case VMX_VMCS_GUEST_LDTR_BASE:
7406 case VMX_VMCS_GUEST_TR_BASE:
7407 case VMX_VMCS_GUEST_GDTR_BASE:
7408 case VMX_VMCS_GUEST_IDTR_BASE:
7409 case VMX_VMCS_GUEST_RSP:
7410 case VMX_VMCS_GUEST_RIP:
7411 case VMX_VMCS_GUEST_SYSENTER_ESP:
7412 case VMX_VMCS_GUEST_SYSENTER_EIP:
7413 {
7414 if (!(RT_HI_U32(u64Val)))
7415 {
7416 /* If this field is 64-bit, VT-x will zero out the top bits. */
7417 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
7418 }
7419 else
7420 {
7421 /* Assert that only the 32->64 switcher case should ever come here. */
7422 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
7423 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
7424 }
7425 break;
7426 }
7427
7428 default:
7429 {
7430 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
7431 pVCpu->hm.s.u32HMError = idxField;
7432 rc = VERR_INVALID_PARAMETER;
7433 break;
7434 }
7435 }
7436 AssertRCReturn(rc, rc);
7437 return rc;
7438}
7439
7440
7441/**
7442 * Queue up a VMWRITE by using the VMCS write cache.
7443 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
7444 *
7445 * @param pVCpu The cross context virtual CPU structure.
7446 * @param idxField The VMCS field encoding.
7447 * @param u64Val 16, 32 or 64-bit value.
7448 */
7449VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
7450{
7451 AssertPtr(pVCpu);
7452 PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
7453
7454 AssertMsgReturn(pCache->Write.cValidEntries < VMX_VMCS_CACHE_MAX_ENTRY - 1,
7455 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
7456
7457 /* Make sure there are no duplicates. */
7458 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
7459 {
7460 if (pCache->Write.aField[i] == idxField)
7461 {
7462 pCache->Write.aFieldVal[i] = u64Val;
7463 return VINF_SUCCESS;
7464 }
7465 }
7466
7467 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
7468 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
7469 pCache->Write.cValidEntries++;
7470 return VINF_SUCCESS;
7471}
7472#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
7473
7474
7475/**
7476 * Sets up the usage of TSC-offsetting and updates the VMCS.
7477 *
7478 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
7479 * VMX-preemption timer.
7480 *
7481 * @returns VBox status code.
7482 * @param pVCpu The cross context virtual CPU structure.
7483 * @param pVmxTransient The VMX-transient structure.
7484 *
7485 * @remarks No-long-jump zone!!!
7486 */
7487static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
7488{
7489 bool fOffsettedTsc;
7490 bool fParavirtTsc;
7491 uint64_t uTscOffset;
7492 PVM pVM = pVCpu->CTX_SUFF(pVM);
7493 PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7494
7495 if (pVM->hm.s.vmx.fUsePreemptTimer)
7496 {
7497 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
7498
7499 /* Make sure the returned values have sane upper and lower boundaries. */
7500 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
7501 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
7502 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
7503 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
7504
7505 /** @todo r=ramshankar: We need to find a way to integrate nested-guest
7506 * preemption timers here. We probably need to clamp the preemption timer,
7507 * after converting the timer value to the host. */
7508 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
7509 int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
7510 AssertRC(rc);
7511 }
7512 else
7513 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
7514
7515 if (fParavirtTsc)
7516 {
7517 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
7518 information before every VM-entry, hence disable it for performance sake. */
7519#if 0
7520 int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
7521 AssertRC(rc);
7522#endif
7523 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
7524 }
7525
7526 uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
7527 if ( fOffsettedTsc
7528 && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
7529 {
7530 if (pVmxTransient->fIsNestedGuest)
7531 uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
7532 if (pVmcsInfo->u64TscOffset != uTscOffset)
7533 {
7534 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
7535 AssertRC(rc);
7536 pVmcsInfo->u64TscOffset = uTscOffset;
7537 }
7538
7539 if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
7540 {
7541 uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
7542 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
7543 AssertRC(rc);
7544 pVmcsInfo->u32ProcCtls = uProcCtls;
7545 }
7546 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
7547 }
7548 else
7549 {
7550 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
7551 if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
7552 {
7553 uProcCtls |= VMX_PROC_CTLS_RDTSC_EXIT;
7554 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
7555 AssertRC(rc);
7556 pVmcsInfo->u32ProcCtls = uProcCtls;
7557 }
7558 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
7559 }
7560}
7561
7562
7563/**
7564 * Gets the IEM exception flags for the specified vector and IDT vectoring /
7565 * VM-exit interruption info type.
7566 *
7567 * @returns The IEM exception flags.
7568 * @param uVector The event vector.
7569 * @param uVmxEventType The VMX event type.
7570 *
7571 * @remarks This function currently only constructs flags required for
7572 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
7573 * and CR2 aspects of an exception are not included).
7574 */
7575static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
7576{
7577 uint32_t fIemXcptFlags;
7578 switch (uVmxEventType)
7579 {
7580 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7581 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7582 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
7583 break;
7584
7585 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7586 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
7587 break;
7588
7589 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7590 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
7591 break;
7592
7593 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
7594 {
7595 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
7596 if (uVector == X86_XCPT_BP)
7597 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
7598 else if (uVector == X86_XCPT_OF)
7599 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
7600 else
7601 {
7602 fIemXcptFlags = 0;
7603 AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
7604 }
7605 break;
7606 }
7607
7608 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7609 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
7610 break;
7611
7612 default:
7613 fIemXcptFlags = 0;
7614 AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
7615 break;
7616 }
7617 return fIemXcptFlags;
7618}
7619
7620
7621/**
7622 * Sets an event as a pending event to be injected into the guest.
7623 *
7624 * @param pVCpu The cross context virtual CPU structure.
7625 * @param u32IntInfo The VM-entry interruption-information field.
7626 * @param cbInstr The VM-entry instruction length in bytes (for software
7627 * interrupts, exceptions and privileged software
7628 * exceptions).
7629 * @param u32ErrCode The VM-entry exception error code.
7630 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
7631 * page-fault.
7632 */
7633DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7634 RTGCUINTPTR GCPtrFaultAddress)
7635{
7636 Assert(!pVCpu->hm.s.Event.fPending);
7637 pVCpu->hm.s.Event.fPending = true;
7638 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
7639 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
7640 pVCpu->hm.s.Event.cbInstr = cbInstr;
7641 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
7642}
7643
7644
7645/**
7646 * Sets an external interrupt as pending-for-injection into the VM.
7647 *
7648 * @param pVCpu The cross context virtual CPU structure.
7649 * @param u8Interrupt The external interrupt vector.
7650 */
7651DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPU pVCpu, uint8_t u8Interrupt)
7652{
7653 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
7654 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
7655 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7656 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7657 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7658}
7659
7660
7661/**
7662 * Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
7663 *
7664 * @param pVCpu The cross context virtual CPU structure.
7665 */
7666DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPU pVCpu)
7667{
7668 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
7669 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
7670 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7671 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7672 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7673}
7674
7675
7676/**
7677 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
7678 *
7679 * @param pVCpu The cross context virtual CPU structure.
7680 */
7681DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
7682{
7683 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
7684 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7685 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7686 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7687 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7688}
7689
7690
7691/**
7692 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7693 *
7694 * @param pVCpu The cross context virtual CPU structure.
7695 */
7696DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
7697{
7698 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
7699 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7700 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7701 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7702 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7703}
7704
7705
7706/**
7707 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7708 *
7709 * @param pVCpu The cross context virtual CPU structure.
7710 */
7711DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
7712{
7713 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
7714 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7715 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7716 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7717 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7718}
7719
7720
7721#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7722/**
7723 * Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
7724 *
7725 * @param pVCpu The cross context virtual CPU structure.
7726 * @param u32ErrCode The error code for the general-protection exception.
7727 */
7728DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
7729{
7730 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
7731 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7732 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7733 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7734 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrCode, 0 /* GCPtrFaultAddress */);
7735}
7736
7737
7738/**
7739 * Sets a stack (\#SS) exception as pending-for-injection into the VM.
7740 *
7741 * @param pVCpu The cross context virtual CPU structure.
7742 * @param u32ErrCode The error code for the stack exception.
7743 */
7744DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
7745{
7746 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
7747 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7748 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7749 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7750 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrCode, 0 /* GCPtrFaultAddress */);
7751}
7752
7753
7754/**
7755 * Decodes the memory operand of an instruction that caused a VM-exit.
7756 *
7757 * The Exit qualification field provides the displacement field for memory
7758 * operand instructions, if any.
7759 *
7760 * @returns Strict VBox status code (i.e. informational status codes too).
7761 * @retval VINF_SUCCESS if the operand was successfully decoded.
7762 * @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
7763 * operand.
7764 * @param pVCpu The cross context virtual CPU structure.
7765 * @param uExitInstrInfo The VM-exit instruction information field.
7766 * @param enmMemAccess The memory operand's access type (read or write).
7767 * @param GCPtrDisp The instruction displacement field, if any. For
7768 * RIP-relative addressing pass RIP + displacement here.
7769 * @param pGCPtrMem Where to store the effective destination memory address.
7770 *
7771 * @remarks Warning! This function ASSUMES the instruction cannot be used in real or
7772 * virtual-8086 mode hence skips those checks while verifying if the
7773 * segment is valid.
7774 */
7775static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
7776 PRTGCPTR pGCPtrMem)
7777{
7778 Assert(pGCPtrMem);
7779 Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
7780 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_EFER
7781 | CPUMCTX_EXTRN_CR0);
7782
7783 static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
7784 static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
7785 AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
7786
7787 VMXEXITINSTRINFO ExitInstrInfo;
7788 ExitInstrInfo.u = uExitInstrInfo;
7789 uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
7790 uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
7791 bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
7792 uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
7793 uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
7794 bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
7795 uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
7796 bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
7797 bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
7798
7799 /*
7800 * Validate instruction information.
7801 * This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
7802 */
7803 AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
7804 ("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
7805 AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
7806 ("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
7807 AssertLogRelMsgReturn(fIsMemOperand,
7808 ("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
7809
7810 /*
7811 * Compute the complete effective address.
7812 *
7813 * See AMD instruction spec. 1.4.2 "SIB Byte Format"
7814 * See AMD spec. 4.5.2 "Segment Registers".
7815 */
7816 RTGCPTR GCPtrMem = GCPtrDisp;
7817 if (fBaseRegValid)
7818 GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
7819 if (fIdxRegValid)
7820 GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
7821
7822 RTGCPTR const GCPtrOff = GCPtrMem;
7823 if ( !fIsLongMode
7824 || iSegReg >= X86_SREG_FS)
7825 GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
7826 GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
7827
7828 /*
7829 * Validate effective address.
7830 * See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
7831 */
7832 uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
7833 Assert(cbAccess > 0);
7834 if (fIsLongMode)
7835 {
7836 if (X86_IS_CANONICAL(GCPtrMem))
7837 {
7838 *pGCPtrMem = GCPtrMem;
7839 return VINF_SUCCESS;
7840 }
7841
7842 /** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
7843 * "Data Limit Checks in 64-bit Mode". */
7844 Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
7845 hmR0VmxSetPendingXcptGP(pVCpu, 0);
7846 return VINF_HM_PENDING_XCPT;
7847 }
7848
7849 /*
7850 * This is a watered down version of iemMemApplySegment().
7851 * Parts that are not applicable for VMX instructions like real-or-v8086 mode
7852 * and segment CPL/DPL checks are skipped.
7853 */
7854 RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
7855 RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
7856 PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7857
7858 /* Check if the segment is present and usable. */
7859 if ( pSel->Attr.n.u1Present
7860 && !pSel->Attr.n.u1Unusable)
7861 {
7862 Assert(pSel->Attr.n.u1DescType);
7863 if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
7864 {
7865 /* Check permissions for the data segment. */
7866 if ( enmMemAccess == VMXMEMACCESS_WRITE
7867 && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
7868 {
7869 Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
7870 hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
7871 return VINF_HM_PENDING_XCPT;
7872 }
7873
7874 /* Check limits if it's a normal data segment. */
7875 if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
7876 {
7877 if ( GCPtrFirst32 > pSel->u32Limit
7878 || GCPtrLast32 > pSel->u32Limit)
7879 {
7880 Log4Func(("Data segment limit exceeded. "
7881 "iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
7882 GCPtrLast32, pSel->u32Limit));
7883 if (iSegReg == X86_SREG_SS)
7884 hmR0VmxSetPendingXcptSS(pVCpu, 0);
7885 else
7886 hmR0VmxSetPendingXcptGP(pVCpu, 0);
7887 return VINF_HM_PENDING_XCPT;
7888 }
7889 }
7890 else
7891 {
7892 /* Check limits if it's an expand-down data segment.
7893 Note! The upper boundary is defined by the B bit, not the G bit! */
7894 if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
7895 || GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
7896 {
7897 Log4Func(("Expand-down data segment limit exceeded. "
7898 "iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
7899 GCPtrLast32, pSel->u32Limit));
7900 if (iSegReg == X86_SREG_SS)
7901 hmR0VmxSetPendingXcptSS(pVCpu, 0);
7902 else
7903 hmR0VmxSetPendingXcptGP(pVCpu, 0);
7904 return VINF_HM_PENDING_XCPT;
7905 }
7906 }
7907 }
7908 else
7909 {
7910 /* Check permissions for the code segment. */
7911 if ( enmMemAccess == VMXMEMACCESS_WRITE
7912 || ( enmMemAccess == VMXMEMACCESS_READ
7913 && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
7914 {
7915 Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
7916 Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
7917 hmR0VmxSetPendingXcptGP(pVCpu, 0);
7918 return VINF_HM_PENDING_XCPT;
7919 }
7920
7921 /* Check limits for the code segment (normal/expand-down not applicable for code segments). */
7922 if ( GCPtrFirst32 > pSel->u32Limit
7923 || GCPtrLast32 > pSel->u32Limit)
7924 {
7925 Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
7926 GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
7927 if (iSegReg == X86_SREG_SS)
7928 hmR0VmxSetPendingXcptSS(pVCpu, 0);
7929 else
7930 hmR0VmxSetPendingXcptGP(pVCpu, 0);
7931 return VINF_HM_PENDING_XCPT;
7932 }
7933 }
7934 }
7935 else
7936 {
7937 Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
7938 hmR0VmxSetPendingXcptGP(pVCpu, 0);
7939 return VINF_HM_PENDING_XCPT;
7940 }
7941
7942 *pGCPtrMem = GCPtrMem;
7943 return VINF_SUCCESS;
7944}
7945
7946
7947/**
7948 * Perform the relevant VMX instruction checks for VM-exits that occurred due to the
7949 * guest attempting to execute a VMX instruction.
7950 *
7951 * @returns Strict VBox status code (i.e. informational status codes too).
7952 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
7953 * @retval VINF_HM_PENDING_XCPT if an exception was raised.
7954 *
7955 * @param pVCpu The cross context virtual CPU structure.
7956 * @param uExitReason The VM-exit reason.
7957 *
7958 * @todo NSTVMX: Document other error codes when VM-exit is implemented.
7959 * @remarks No-long-jump zone!!!
7960 */
7961static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
7962{
7963 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS
7964 | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_EFER);
7965
7966 /*
7967 * The physical CPU would have already checked the CPU mode/code segment.
7968 * We shall just assert here for paranoia.
7969 * See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
7970 */
7971 Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
7972 Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
7973 || CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
7974
7975 if (uExitReason == VMX_EXIT_VMXON)
7976 {
7977 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
7978
7979 /*
7980 * We check CR4.VMXE because it is required to be always set while in VMX operation
7981 * by physical CPUs and our CR4 read-shadow is only consulted when executing specific
7982 * instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
7983 * otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
7984 */
7985 if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
7986 {
7987 Log4Func(("CR4.VMXE is not set -> #UD\n"));
7988 hmR0VmxSetPendingXcptUD(pVCpu);
7989 return VINF_HM_PENDING_XCPT;
7990 }
7991 }
7992 else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
7993 {
7994 /*
7995 * The guest has not entered VMX operation but attempted to execute a VMX instruction
7996 * (other than VMXON), we need to raise a #UD.
7997 */
7998 Log4Func(("Not in VMX root mode -> #UD\n"));
7999 hmR0VmxSetPendingXcptUD(pVCpu);
8000 return VINF_HM_PENDING_XCPT;
8001 }
8002
8003 /* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
8004 return VINF_SUCCESS;
8005}
8006#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
8007
8008
8009static void hmR0VmxFixUnusableSegRegAttr(PVMCPU pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
8010{
8011 Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
8012
8013 /*
8014 * If VT-x marks the segment as unusable, most other bits remain undefined:
8015 * - For CS the L, D and G bits have meaning.
8016 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
8017 * - For the remaining data segments no bits are defined.
8018 *
8019 * The present bit and the unusable bit has been observed to be set at the
8020 * same time (the selector was supposed to be invalid as we started executing
8021 * a V8086 interrupt in ring-0).
8022 *
8023 * What should be important for the rest of the VBox code, is that the P bit is
8024 * cleared. Some of the other VBox code recognizes the unusable bit, but
8025 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
8026 * safe side here, we'll strip off P and other bits we don't care about. If
8027 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
8028 *
8029 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
8030 */
8031#ifdef VBOX_STRICT
8032 uint32_t const uAttr = pSelReg->Attr.u;
8033#endif
8034
8035 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
8036 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
8037 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
8038
8039#ifdef VBOX_STRICT
8040 VMMRZCallRing3Disable(pVCpu);
8041 Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
8042# ifdef DEBUG_bird
8043 AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
8044 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
8045 idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
8046# endif
8047 VMMRZCallRing3Enable(pVCpu);
8048 NOREF(uAttr);
8049#endif
8050 RT_NOREF2(pVCpu, idxSel);
8051}
8052
8053
8054/**
8055 * Imports a guest segment register from the current VMCS into the guest-CPU
8056 * context.
8057 *
8058 * @returns VBox status code.
8059 * @param pVCpu The cross context virtual CPU structure.
8060 * @param iSegReg The segment register number (X86_SREG_XXX).
8061 *
8062 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
8063 * do not log!
8064 */
8065static int hmR0VmxImportGuestSegReg(PVMCPU pVCpu, uint8_t iSegReg)
8066{
8067 Assert(iSegReg < X86_SREG_COUNT);
8068
8069 uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
8070 uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
8071 uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
8072#ifdef VMX_USE_CACHED_VMCS_ACCESSES
8073 uint32_t const idxBase = g_aVmcsCacheSegBase[iSegReg];
8074#else
8075 uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
8076#endif
8077 uint64_t u64Base;
8078 uint32_t u32Sel, u32Limit, u32Attr;
8079 int rc = VMXReadVmcs32(idxSel, &u32Sel);
8080 rc |= VMXReadVmcs32(idxLimit, &u32Limit);
8081 rc |= VMXReadVmcs32(idxAttr, &u32Attr);
8082 rc |= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
8083 if (RT_SUCCESS(rc))
8084 {
8085 PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
8086 pSelReg->Sel = u32Sel;
8087 pSelReg->ValidSel = u32Sel;
8088 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
8089 pSelReg->u32Limit = u32Limit;
8090 pSelReg->u64Base = u64Base;
8091 pSelReg->Attr.u = u32Attr;
8092 if (u32Attr & X86DESCATTR_UNUSABLE)
8093 hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
8094 }
8095 return rc;
8096}
8097
8098
8099/**
8100 * Imports the guest LDTR from the current VMCS into the guest-CPU context.
8101 *
8102 * @returns VBox status code.
8103 * @param pVCpu The cross context virtual CPU structure.
8104 *
8105 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
8106 * do not log!
8107 */
8108static int hmR0VmxImportGuestLdtr(PVMCPU pVCpu)
8109{
8110 uint64_t u64Base;
8111 uint32_t u32Sel, u32Limit, u32Attr;
8112 int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, &u32Sel);
8113 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit);
8114 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);
8115 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, &u64Base);
8116 if (RT_SUCCESS(rc))
8117 {
8118 pVCpu->cpum.GstCtx.ldtr.Sel = u32Sel;
8119 pVCpu->cpum.GstCtx.ldtr.ValidSel = u32Sel;
8120 pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
8121 pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
8122 pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
8123 pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
8124 if (u32Attr & X86DESCATTR_UNUSABLE)
8125 hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
8126 }
8127 return rc;
8128}
8129
8130
8131/**
8132 * Imports the guest TR from the current VMCS into the guest-CPU context.
8133 *
8134 * @returns VBox status code.
8135 * @param pVCpu The cross context virtual CPU structure.
8136 *
8137 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
8138 * do not log!
8139 */
8140static int hmR0VmxImportGuestTr(PVMCPU pVCpu)
8141{
8142 uint32_t u32Sel, u32Limit, u32Attr;
8143 uint64_t u64Base;
8144 int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_TR_SEL, &u32Sel);
8145 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit);
8146 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);
8147 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_TR_BASE, &u64Base);
8148 AssertRCReturn(rc, rc);
8149
8150 pVCpu->cpum.GstCtx.tr.Sel = u32Sel;
8151 pVCpu->cpum.GstCtx.tr.ValidSel = u32Sel;
8152 pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
8153 pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
8154 pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
8155 pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
8156 /* TR is the only selector that can never be unusable. */
8157 Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
8158 return VINF_SUCCESS;
8159}
8160
8161
8162/**
8163 * Imports the guest RIP from the VMCS back into the guest-CPU context.
8164 *
8165 * @returns VBox status code.
8166 * @param pVCpu The cross context virtual CPU structure.
8167 *
8168 * @remarks Called with interrupts and/or preemption disabled, should not assert!
8169 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
8170 * instead!!!
8171 */
8172static int hmR0VmxImportGuestRip(PVMCPU pVCpu)
8173{
8174 uint64_t u64Val;
8175 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8176 if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
8177 {
8178 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
8179 if (RT_SUCCESS(rc))
8180 {
8181 pCtx->rip = u64Val;
8182 EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
8183 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
8184 }
8185 return rc;
8186 }
8187 return VINF_SUCCESS;
8188}
8189
8190
8191/**
8192 * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
8193 *
8194 * @returns VBox status code.
8195 * @param pVCpu The cross context virtual CPU structure.
8196 * @param pVmcsInfo The VMCS info. object.
8197 *
8198 * @remarks Called with interrupts and/or preemption disabled, should not assert!
8199 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
8200 * instead!!!
8201 */
8202static int hmR0VmxImportGuestRFlags(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
8203{
8204 uint32_t u32Val;
8205 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8206 if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
8207 {
8208 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
8209 if (RT_SUCCESS(rc))
8210 {
8211 pCtx->eflags.u32 = u32Val;
8212
8213 /* Restore eflags for real-on-v86-mode hack. */
8214 if (pVmcsInfo->RealMode.fRealOnV86Active)
8215 {
8216 pCtx->eflags.Bits.u1VM = 0;
8217 pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
8218 }
8219 }
8220 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
8221 return rc;
8222 }
8223 return VINF_SUCCESS;
8224}
8225
8226
8227/**
8228 * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
8229 * context.
8230 *
8231 * @returns VBox status code.
8232 * @param pVCpu The cross context virtual CPU structure.
8233 * @param pVmcsInfo The VMCS info. object.
8234 *
8235 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
8236 * do not log!
8237 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
8238 * instead!!!
8239 */
8240static int hmR0VmxImportGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
8241{
8242 uint32_t u32Val;
8243 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
8244 if (RT_SUCCESS(rc))
8245 {
8246 if (!u32Val)
8247 {
8248 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
8249 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
8250
8251 CPUMSetGuestNmiBlocking(pVCpu, false);
8252 }
8253 else
8254 {
8255 /*
8256 * We must import RIP here to set our EM interrupt-inhibited state.
8257 * We also import RFLAGS as our code that evaluates pending interrupts
8258 * before VM-entry requires it.
8259 */
8260 rc = hmR0VmxImportGuestRip(pVCpu);
8261 rc |= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
8262 if (RT_SUCCESS(rc))
8263 {
8264 if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS | VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
8265 EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
8266 else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
8267 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
8268
8269 bool const fNmiBlocking = RT_BOOL(u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8270 CPUMSetGuestNmiBlocking(pVCpu, fNmiBlocking);
8271 }
8272 }
8273 }
8274 return rc;
8275}
8276
8277
8278/**
8279 * Worker for VMXR0ImportStateOnDemand.
8280 *
8281 * @returns VBox status code.
8282 * @param pVCpu The cross context virtual CPU structure.
8283 * @param pVmcsInfo The VMCS info. object.
8284 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
8285 */
8286static int hmR0VmxImportGuestState(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
8287{
8288#define VMXLOCAL_BREAK_RC(a_rc) \
8289 if (RT_SUCCESS(a_rc)) \
8290 { } \
8291 else \
8292 break
8293
8294 int rc = VINF_SUCCESS;
8295 PVM pVM = pVCpu->CTX_SUFF(pVM);
8296 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8297 uint64_t u64Val;
8298 uint32_t u32Val;
8299
8300 /*
8301 * Note! This is hack to workaround a mysterious BSOD observed with release builds
8302 * on Windows 10 64-bit hosts. Profile and debug builds are not affected and
8303 * neither are other host platforms.
8304 *
8305 * Committing this temporarily as it prevents BSOD.
8306 *
8307 * Update: This is very likely a compiler optimization bug, see @bugref{9180}.
8308 */
8309#ifdef RT_OS_WINDOWS
8310 if (pVM == 0 || pVM == (void *)(uintptr_t)-1)
8311 return VERR_HM_IPE_1;
8312#endif
8313
8314 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
8315
8316 /*
8317 * We disable interrupts to make the updating of the state and in particular
8318 * the fExtrn modification atomic wrt to preemption hooks.
8319 */
8320 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
8321
8322 fWhat &= pCtx->fExtrn;
8323 if (fWhat)
8324 {
8325 do
8326 {
8327 if (fWhat & CPUMCTX_EXTRN_RIP)
8328 {
8329 rc = hmR0VmxImportGuestRip(pVCpu);
8330 VMXLOCAL_BREAK_RC(rc);
8331 }
8332
8333 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
8334 {
8335 rc = hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
8336 VMXLOCAL_BREAK_RC(rc);
8337 }
8338
8339 if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
8340 {
8341 rc = hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
8342 VMXLOCAL_BREAK_RC(rc);
8343 }
8344
8345 if (fWhat & CPUMCTX_EXTRN_RSP)
8346 {
8347 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
8348 VMXLOCAL_BREAK_RC(rc);
8349 pCtx->rsp = u64Val;
8350 }
8351
8352 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
8353 {
8354 bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
8355 if (fWhat & CPUMCTX_EXTRN_CS)
8356 {
8357 rc |= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
8358 rc |= hmR0VmxImportGuestRip(pVCpu);
8359 if (fRealOnV86Active)
8360 pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
8361 EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
8362 }
8363 if (fWhat & CPUMCTX_EXTRN_SS)
8364 {
8365 rc |= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
8366 if (fRealOnV86Active)
8367 pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
8368 }
8369 if (fWhat & CPUMCTX_EXTRN_DS)
8370 {
8371 rc |= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
8372 if (fRealOnV86Active)
8373 pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
8374 }
8375 if (fWhat & CPUMCTX_EXTRN_ES)
8376 {
8377 rc |= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
8378 if (fRealOnV86Active)
8379 pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
8380 }
8381 if (fWhat & CPUMCTX_EXTRN_FS)
8382 {
8383 rc |= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
8384 if (fRealOnV86Active)
8385 pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
8386 }
8387 if (fWhat & CPUMCTX_EXTRN_GS)
8388 {
8389 rc |= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
8390 if (fRealOnV86Active)
8391 pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
8392 }
8393 VMXLOCAL_BREAK_RC(rc);
8394 }
8395
8396 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
8397 {
8398 if (fWhat & CPUMCTX_EXTRN_LDTR)
8399 rc |= hmR0VmxImportGuestLdtr(pVCpu);
8400
8401 if (fWhat & CPUMCTX_EXTRN_GDTR)
8402 {
8403 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8404 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8405 pCtx->gdtr.pGdt = u64Val;
8406 pCtx->gdtr.cbGdt = u32Val;
8407 }
8408
8409 /* Guest IDTR. */
8410 if (fWhat & CPUMCTX_EXTRN_IDTR)
8411 {
8412 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8413 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8414 pCtx->idtr.pIdt = u64Val;
8415 pCtx->idtr.cbIdt = u32Val;
8416 }
8417
8418 /* Guest TR. */
8419 if (fWhat & CPUMCTX_EXTRN_TR)
8420 {
8421 /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
8422 don't need to import that one. */
8423 if (!pVmcsInfo->RealMode.fRealOnV86Active)
8424 rc |= hmR0VmxImportGuestTr(pVCpu);
8425 }
8426 VMXLOCAL_BREAK_RC(rc);
8427 }
8428
8429 if (fWhat & CPUMCTX_EXTRN_DR7)
8430 {
8431 if (!pVCpu->hm.s.fUsingHyperDR7)
8432 {
8433 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
8434 rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
8435 VMXLOCAL_BREAK_RC(rc);
8436 pCtx->dr[7] = u32Val;
8437 }
8438 }
8439
8440 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
8441 {
8442 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
8443 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
8444 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
8445 pCtx->SysEnter.cs = u32Val;
8446 VMXLOCAL_BREAK_RC(rc);
8447 }
8448
8449#if HC_ARCH_BITS == 64
8450 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
8451 {
8452 if ( pVM->hm.s.fAllow64BitGuests
8453 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
8454 pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
8455 }
8456
8457 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
8458 {
8459 if ( pVM->hm.s.fAllow64BitGuests
8460 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
8461 {
8462 pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
8463 pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
8464 pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
8465 }
8466 }
8467#endif
8468
8469 if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
8470#if HC_ARCH_BITS == 32
8471 || (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS))
8472#endif
8473 )
8474 {
8475 PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
8476 uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
8477 Assert(pMsrs);
8478 Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
8479 Assert(sizeof(*pMsrs) * cMsrs <= X86_PAGE_4K_SIZE);
8480 for (uint32_t i = 0; i < cMsrs; i++)
8481 {
8482 uint32_t const idMsr = pMsrs[i].u32Msr;
8483 switch (idMsr)
8484 {
8485 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
8486 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
8487 case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
8488#if HC_ARCH_BITS == 32
8489 case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsrs[i].u64Value; break;
8490 case MSR_K6_STAR: pCtx->msrSTAR = pMsrs[i].u64Value; break;
8491 case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsrs[i].u64Value; break;
8492 case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsrs[i].u64Value; break;
8493#endif
8494 default:
8495 {
8496 pCtx->fExtrn = 0;
8497 pVCpu->hm.s.u32HMError = pMsrs->u32Msr;
8498 ASMSetFlags(fEFlags);
8499 AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
8500 return VERR_HM_UNEXPECTED_LD_ST_MSR;
8501 }
8502 }
8503 }
8504 }
8505
8506 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
8507 {
8508 uint64_t u64Shadow;
8509 if (fWhat & CPUMCTX_EXTRN_CR0)
8510 {
8511 /** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
8512 * remove when we drop 32-bit host w/ 64-bit host support, see
8513 * @bugref{9180#c39}. */
8514 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
8515#if HC_ARCH_BITS == 32
8516 uint32_t u32Shadow;
8517 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
8518 u64Shadow = u32Shadow;
8519#else
8520 rc |= VMXReadVmcs64(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);
8521#endif
8522 VMXLOCAL_BREAK_RC(rc);
8523 u64Val = u32Val;
8524 u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
8525 | (u64Shadow & pVmcsInfo->u64Cr0Mask);
8526#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8527 /*
8528 * Reapply the nested-guest's CR0 fixed bits that might have been altered while
8529 * exporting the nested-guest CR0 for executing using hardware-assisted VMX.
8530 */
8531 if (CPUMIsGuestInVmxNonRootMode(pCtx))
8532 {
8533 u64Val |= pCtx->hwvirt.vmx.Msrs.u64Cr0Fixed0;
8534 u64Val &= pCtx->hwvirt.vmx.Msrs.u64Cr0Fixed1;
8535 }
8536#endif
8537 VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
8538 CPUMSetGuestCR0(pVCpu, u64Val);
8539 VMMRZCallRing3Enable(pVCpu);
8540 }
8541
8542 if (fWhat & CPUMCTX_EXTRN_CR4)
8543 {
8544 /** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
8545 * remove when we drop 32-bit host w/ 64-bit host support, see
8546 * @bugref{9180#c39}. */
8547 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
8548#if HC_ARCH_BITS == 32
8549 uint32_t u32Shadow;
8550 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
8551 u64Shadow = u32Shadow;
8552#else
8553 rc |= VMXReadVmcs64(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);
8554#endif
8555 VMXLOCAL_BREAK_RC(rc);
8556 u64Val = u32Val;
8557 u64Val = (u64Val & ~pVmcsInfo->u64Cr4Mask)
8558 | (u64Shadow & pVmcsInfo->u64Cr4Mask);
8559#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8560 /*
8561 * Reapply the nested-guest's CR4 fixed bits that might have been altered while
8562 * exporting the nested-guest CR4 for executing using hardware-assisted VMX.
8563 */
8564 if (CPUMIsGuestInVmxNonRootMode(pCtx))
8565 {
8566 u64Val |= pCtx->hwvirt.vmx.Msrs.u64Cr4Fixed0;
8567 u64Val &= pCtx->hwvirt.vmx.Msrs.u64Cr4Fixed1;
8568 }
8569#endif
8570 pCtx->cr4 = u64Val;
8571 }
8572
8573 if (fWhat & CPUMCTX_EXTRN_CR3)
8574 {
8575 /* CR0.PG bit changes are always intercepted, so it's up to date. */
8576 if ( pVM->hm.s.vmx.fUnrestrictedGuest
8577 || ( pVM->hm.s.fNestedPaging
8578 && CPUMIsGuestPagingEnabledEx(pCtx)))
8579 {
8580 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
8581 VMXLOCAL_BREAK_RC(rc);
8582 if (pCtx->cr3 != u64Val)
8583 {
8584 pCtx->cr3 = u64Val;
8585 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
8586 }
8587
8588 /* If the guest is in PAE mode, sync back the PDPE's into the guest state.
8589 Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
8590 if (CPUMIsGuestInPAEModeEx(pCtx))
8591 {
8592 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
8593 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
8594 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
8595 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
8596 VMXLOCAL_BREAK_RC(rc);
8597 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
8598 }
8599 }
8600 }
8601 }
8602
8603#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8604 if (fWhat & CPUMCTX_EXTRN_HWVIRT)
8605 {
8606 if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
8607 && !CPUMIsGuestInVmxNonRootMode(pCtx))
8608 {
8609 Assert(CPUMIsGuestInVmxRootMode(pCtx));
8610 rc = hmR0VmxCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo);
8611 VMXLOCAL_BREAK_RC(rc);
8612 }
8613
8614# if 0
8615 /** @todo NSTVMX: We handle most of these fields individually by passing it to IEM
8616 * VM-exit handlers as parameters. We would handle it differently when using
8617 * the fast path. */
8618 /*
8619 * The hardware virtualization state currently consists of VMCS fields that may be
8620 * modified by execution of the nested-guest (that are not part of the general
8621 * guest state) and is visible to guest software. Hence, it is technically part of
8622 * the guest-CPU state when executing a nested-guest.
8623 */
8624 if (CPUMIsGuestInVmxNonRootMode(pCtx))
8625 {
8626 PVMXVVMCS pGstVmcs = pCtx->hwvirt.vmx.CTX_SUFF(pVmcs);
8627 rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pGstVmcs->u32RoExitReason);
8628 rc |= VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pGstVmcs->u64RoExitQual.u);
8629 VMXLOCAL_BREAK_RC(rc);
8630
8631 /*
8632 * VM-entry can fail due to invalid-guest state, machine-check events and
8633 * MSR loading failures. Other than VM-exit reason and Exit qualification
8634 * all other VMCS fields are left unmodified on VM-entry failure.
8635 *
8636 * See Intel spec. 26.7 "VM-entry Failures During Or After Loading Guest State".
8637 */
8638 bool const fEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(pGstVmcs->u32RoExitReason);
8639 if (!fEntryFailed)
8640 {
8641 /*
8642 * Some notes on VMCS fields that may need importing when the fast path
8643 * is implemented. Currently we fully emulate VMLAUNCH/VMRESUME in IEM.
8644 *
8645 * Requires fixing up when using hardware-assisted VMX:
8646 * - VM-exit interruption info: Shouldn't reflect host interrupts/NMIs.
8647 * - VM-exit interruption error code: Cleared to 0 when not appropriate.
8648 * - IDT-vectoring info: Think about this.
8649 * - IDT-vectoring error code: Think about this.
8650 *
8651 * Emulated:
8652 * - Guest-interruptiblity state: Derived from FFs and RIP.
8653 * - Guest pending debug exceptions: Derived from DR6.
8654 * - Guest activity state: Emulated from EM state.
8655 * - Guest PDPTEs: Currently all 0s since we don't support nested EPT.
8656 * - Entry-interrupt info: Emulated, cleared to 0.
8657 */
8658 rc |= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pGstVmcs->u32RoExitIntInfo);
8659 rc |= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pGstVmcs->u32RoExitIntErrCode);
8660 rc |= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pGstVmcs->u32RoIdtVectoringInfo);
8661 rc |= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pGstVmcs->u32RoIdtVectoringErrCode);
8662 rc |= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pGstVmcs->u32RoExitInstrLen);
8663 rc |= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pGstVmcs->u32RoExitIntInfo);
8664 rc |= VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pGstVmcs->u64RoGuestPhysAddr.u);
8665 rc |= VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pGstVmcs->u64RoGuestLinearAddr.u);
8666 /** @todo NSTVMX: Save and adjust preemption timer value. */
8667 }
8668
8669 VMXLOCAL_BREAK_RC(rc);
8670 }
8671# endif
8672 }
8673#endif
8674 } while (0);
8675
8676 if (RT_SUCCESS(rc))
8677 {
8678 /* Update fExtrn. */
8679 pCtx->fExtrn &= ~fWhat;
8680
8681 /* If everything has been imported, clear the HM keeper bit. */
8682 if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
8683 {
8684 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
8685 Assert(!pCtx->fExtrn);
8686 }
8687 }
8688 }
8689 else
8690 AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
8691
8692 ASMSetFlags(fEFlags);
8693
8694 STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
8695
8696 if (RT_SUCCESS(rc))
8697 { /* likely */ }
8698 else
8699 return rc;
8700
8701 /*
8702 * Honor any pending CR3 updates.
8703 *
8704 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
8705 * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
8706 * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
8707 *
8708 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
8709 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
8710 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
8711 * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
8712 *
8713 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
8714 */
8715 if (VMMRZCallRing3IsEnabled(pVCpu))
8716 {
8717 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8718 {
8719 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
8720 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8721 }
8722
8723 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8724 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8725
8726 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8727 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8728 }
8729
8730 return VINF_SUCCESS;
8731#undef VMXLOCAL_BREAK_RC
8732}
8733
8734
8735/**
8736 * Saves the guest state from the VMCS into the guest-CPU context.
8737 *
8738 * @returns VBox status code.
8739 * @param pVCpu The cross context virtual CPU structure.
8740 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
8741 */
8742VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
8743{
8744 AssertPtr(pVCpu);
8745 PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8746 return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
8747}
8748
8749
8750/**
8751 * Check per-VM and per-VCPU force flag actions that require us to go back to
8752 * ring-3 for one reason or another.
8753 *
8754 * @returns Strict VBox status code (i.e. informational status codes too)
8755 * @retval VINF_SUCCESS if we don't have any actions that require going back to
8756 * ring-3.
8757 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
8758 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
8759 * interrupts)
8760 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
8761 * all EMTs to be in ring-3.
8762 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
8763 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
8764 * to the EM loop.
8765 *
8766 * @param pVCpu The cross context virtual CPU structure.
8767 * @param fStepping Whether we are single-stepping the guest using the
8768 * hypervisor debugger.
8769 */
8770static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
8771{
8772 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8773
8774 /*
8775 * Update pending interrupts into the APIC's IRR.
8776 */
8777 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
8778 APICUpdatePendingInterrupts(pVCpu);
8779
8780 /*
8781 * Anything pending? Should be more likely than not if we're doing a good job.
8782 */
8783 PVM pVM = pVCpu->CTX_SUFF(pVM);
8784 if ( !fStepping
8785 ? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
8786 && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
8787 : !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
8788 && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
8789 return VINF_SUCCESS;
8790
8791 /* Pending PGM C3 sync. */
8792 if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
8793 {
8794 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8795 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
8796 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
8797 VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
8798 if (rcStrict2 != VINF_SUCCESS)
8799 {
8800 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
8801 Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
8802 return rcStrict2;
8803 }
8804 }
8805
8806 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
8807 if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
8808 || VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8809 {
8810 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8811 int rc2 = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
8812 Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
8813 return rc2;
8814 }
8815
8816 /* Pending VM request packets, such as hardware interrupts. */
8817 if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
8818 || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
8819 {
8820 Log4Func(("Pending VM request forcing us back to ring-3\n"));
8821 return VINF_EM_PENDING_REQUEST;
8822 }
8823
8824 /* Pending PGM pool flushes. */
8825 if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
8826 {
8827 Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
8828 return VINF_PGM_POOL_FLUSH_PENDING;
8829 }
8830
8831 /* Pending DMA requests. */
8832 if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
8833 {
8834 Log4Func(("Pending DMA request forcing us back to ring-3\n"));
8835 return VINF_EM_RAW_TO_R3;
8836 }
8837
8838 return VINF_SUCCESS;
8839}
8840
8841
8842/**
8843 * Converts any TRPM trap into a pending HM event. This is typically used when
8844 * entering from ring-3 (not longjmp returns).
8845 *
8846 * @param pVCpu The cross context virtual CPU structure.
8847 */
8848static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
8849{
8850 Assert(TRPMHasTrap(pVCpu));
8851 Assert(!pVCpu->hm.s.Event.fPending);
8852
8853 uint8_t uVector;
8854 TRPMEVENT enmTrpmEvent;
8855 RTGCUINT uErrCode;
8856 RTGCUINTPTR GCPtrFaultAddress;
8857 uint8_t cbInstr;
8858
8859 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
8860 AssertRC(rc);
8861
8862 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
8863 uint32_t u32IntInfo = uVector | VMX_EXIT_INT_INFO_VALID;
8864 if (enmTrpmEvent == TRPM_TRAP)
8865 {
8866 /** @todo r=ramshankar: TRPM currently offers no way to determine a \#DB that was
8867 * generated using INT1 (ICEBP). */
8868 switch (uVector)
8869 {
8870 case X86_XCPT_NMI:
8871 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8872 break;
8873
8874 case X86_XCPT_BP:
8875 case X86_XCPT_OF:
8876 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8877 break;
8878
8879 case X86_XCPT_PF:
8880 case X86_XCPT_DF:
8881 case X86_XCPT_TS:
8882 case X86_XCPT_NP:
8883 case X86_XCPT_SS:
8884 case X86_XCPT_GP:
8885 case X86_XCPT_AC:
8886 u32IntInfo |= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
8887 RT_FALL_THRU();
8888 default:
8889 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8890 break;
8891 }
8892 }
8893 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
8894 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8895 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
8896 {
8897 switch (uVector)
8898 {
8899 case X86_XCPT_BP:
8900 case X86_XCPT_OF:
8901 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8902 break;
8903
8904 default:
8905 Assert(uVector == X86_XCPT_DB);
8906 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8907 break;
8908 }
8909 }
8910 else
8911 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
8912
8913 rc = TRPMResetTrap(pVCpu);
8914 AssertRC(rc);
8915 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
8916 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
8917
8918 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
8919}
8920
8921
8922/**
8923 * Converts the pending HM event into a TRPM trap.
8924 *
8925 * @param pVCpu The cross context virtual CPU structure.
8926 */
8927static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
8928{
8929 Assert(pVCpu->hm.s.Event.fPending);
8930
8931 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8932 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
8933 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
8934 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
8935
8936 /* If a trap was already pending, we did something wrong! */
8937 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
8938
8939 /** @todo Use HMVmxEventToTrpmEventType() later. */
8940 TRPMEVENT enmTrapType;
8941 switch (uVectorType)
8942 {
8943 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
8944 enmTrapType = TRPM_HARDWARE_INT;
8945 break;
8946
8947 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
8948 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
8949 enmTrapType = TRPM_TRAP;
8950 break;
8951
8952 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT: /* #DB (INT1/ICEBP). */
8953 Assert(uVector == X86_XCPT_DB);
8954 enmTrapType = TRPM_SOFTWARE_INT;
8955 break;
8956
8957 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP (INT3) and #OF (INTO) */
8958 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
8959 enmTrapType = TRPM_SOFTWARE_INT;
8960 break;
8961
8962 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
8963 enmTrapType = TRPM_SOFTWARE_INT;
8964 break;
8965
8966 default:
8967 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
8968 enmTrapType = TRPM_32BIT_HACK;
8969 break;
8970 }
8971
8972 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
8973
8974 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
8975 AssertRC(rc);
8976
8977 if (fErrorCodeValid)
8978 TRPMSetErrorCode(pVCpu, uErrorCode);
8979
8980 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8981 && uVector == X86_XCPT_PF)
8982 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
8983 else if (enmTrapType == TRPM_SOFTWARE_INT)
8984 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
8985
8986 /* We're now done converting the pending event. */
8987 pVCpu->hm.s.Event.fPending = false;
8988}
8989
8990
8991/**
8992 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
8993 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
8994 *
8995 * @param pVCpu The cross context virtual CPU structure.
8996 * @param pVmcsInfo The VMCS info. object.
8997 */
8998static void hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8999{
9000 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
9001 {
9002 if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
9003 {
9004 pVmcsInfo->u32ProcCtls |= VMX_PROC_CTLS_INT_WINDOW_EXIT;
9005 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
9006 AssertRC(rc);
9007 }
9008 } /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
9009}
9010
9011
9012/**
9013 * Clears the interrupt-window exiting control in the VMCS.
9014 *
9015 * @param pVmcsInfo The VMCS info. object.
9016 */
9017DECLINLINE(int) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
9018{
9019 if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
9020 {
9021 pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
9022 return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
9023 }
9024 return VINF_SUCCESS;
9025}
9026
9027
9028/**
9029 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
9030 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
9031 *
9032 * @param pVCpu The cross context virtual CPU structure.
9033 * @param pVmcsInfo The VMCS info. object.
9034 */
9035static void hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
9036{
9037 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
9038 {
9039 if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
9040 {
9041 pVmcsInfo->u32ProcCtls |= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
9042 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
9043 AssertRC(rc);
9044 Log4Func(("Setup NMI-window exiting\n"));
9045 }
9046 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
9047}
9048
9049
9050/**
9051 * Clears the NMI-window exiting control in the VMCS.
9052 *
9053 * @param pVmcsInfo The VMCS info. object.
9054 */
9055DECLINLINE(int) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
9056{
9057 if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
9058 {
9059 pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
9060 return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
9061 }
9062 return VINF_SUCCESS;
9063}
9064
9065
9066/**
9067 * Does the necessary state syncing before returning to ring-3 for any reason
9068 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
9069 *
9070 * @returns VBox status code.
9071 * @param pVCpu The cross context virtual CPU structure.
9072 * @param fImportState Whether to import the guest state from the VMCS back
9073 * to the guest-CPU context.
9074 *
9075 * @remarks No-long-jmp zone!!!
9076 */
9077static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
9078{
9079 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9080 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9081
9082 RTCPUID idCpu = RTMpCpuId();
9083 Log4Func(("HostCpuId=%u\n", idCpu));
9084
9085 /*
9086 * !!! IMPORTANT !!!
9087 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
9088 */
9089
9090 /* Save the guest state if necessary. */
9091 PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9092 if (fImportState)
9093 {
9094 int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
9095 AssertRCReturn(rc, rc);
9096 }
9097
9098 /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
9099 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
9100 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
9101
9102 /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
9103#ifdef VBOX_STRICT
9104 if (CPUMIsHyperDebugStateActive(pVCpu))
9105 Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
9106#endif
9107 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
9108 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
9109 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
9110
9111#if HC_ARCH_BITS == 64
9112 /* Restore host-state bits that VT-x only restores partially. */
9113 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
9114 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
9115 {
9116 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
9117 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
9118 }
9119 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
9120#endif
9121
9122 /* Restore the lazy host MSRs as we're leaving VT-x context. */
9123 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
9124 {
9125 /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
9126 if (!fImportState)
9127 {
9128 int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS);
9129 AssertRCReturn(rc, rc);
9130 }
9131 hmR0VmxLazyRestoreHostMsrs(pVCpu);
9132 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
9133 }
9134 else
9135 pVCpu->hm.s.vmx.fLazyMsrs = 0;
9136
9137 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
9138 pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
9139
9140 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
9141 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
9142 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
9143 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
9144 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
9145 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
9146 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
9147 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
9148 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
9149
9150 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
9151
9152 /** @todo This partially defeats the purpose of having preemption hooks.
9153 * The problem is, deregistering the hooks should be moved to a place that
9154 * lasts until the EMT is about to be destroyed not everytime while leaving HM
9155 * context.
9156 */
9157 int rc = hmR0VmxClearVmcs(pVmcsInfo);
9158 AssertRCReturn(rc, rc);
9159
9160#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9161 /*
9162 * A valid shadow VMCS is made active as part of VM-entry. It is necessary to
9163 * clear a shadow VMCS before allowing that VMCS to become active on another
9164 * logical processor. We may or may not be importing guest state which clears
9165 * it, so cover for it here.
9166 *
9167 * See Intel spec. 24.11.1 "Software Use of Virtual-Machine Control Structures".
9168 */
9169 if ( pVmcsInfo->pvShadowVmcs
9170 && pVmcsInfo->fShadowVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
9171 {
9172 rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
9173 AssertRCReturn(rc, rc);
9174 }
9175#endif
9176
9177 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
9178 NOREF(idCpu);
9179 return VINF_SUCCESS;
9180}
9181
9182
9183/**
9184 * Leaves the VT-x session.
9185 *
9186 * @returns VBox status code.
9187 * @param pVCpu The cross context virtual CPU structure.
9188 *
9189 * @remarks No-long-jmp zone!!!
9190 */
9191static int hmR0VmxLeaveSession(PVMCPU pVCpu)
9192{
9193 HM_DISABLE_PREEMPT(pVCpu);
9194 HMVMX_ASSERT_CPU_SAFE(pVCpu);
9195 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9196 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9197
9198 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
9199 and done this from the VMXR0ThreadCtxCallback(). */
9200 if (!pVCpu->hm.s.fLeaveDone)
9201 {
9202 int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
9203 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
9204 pVCpu->hm.s.fLeaveDone = true;
9205 }
9206 Assert(!pVCpu->cpum.GstCtx.fExtrn);
9207
9208 /*
9209 * !!! IMPORTANT !!!
9210 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
9211 */
9212
9213 /* Deregister hook now that we've left HM context before re-enabling preemption. */
9214 /** @todo Deregistering here means we need to VMCLEAR always
9215 * (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
9216 * for calling VMMR0ThreadCtxHookDisable here! */
9217 VMMR0ThreadCtxHookDisable(pVCpu);
9218
9219 /* Leave HM context. This takes care of local init (term). */
9220 int rc = HMR0LeaveCpu(pVCpu);
9221
9222 HM_RESTORE_PREEMPT();
9223 return rc;
9224}
9225
9226
9227/**
9228 * Does the necessary state syncing before doing a longjmp to ring-3.
9229 *
9230 * @returns VBox status code.
9231 * @param pVCpu The cross context virtual CPU structure.
9232 *
9233 * @remarks No-long-jmp zone!!!
9234 */
9235DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
9236{
9237 return hmR0VmxLeaveSession(pVCpu);
9238}
9239
9240
9241/**
9242 * Take necessary actions before going back to ring-3.
9243 *
9244 * An action requires us to go back to ring-3. This function does the necessary
9245 * steps before we can safely return to ring-3. This is not the same as longjmps
9246 * to ring-3, this is voluntary and prepares the guest so it may continue
9247 * executing outside HM (recompiler/IEM).
9248 *
9249 * @returns VBox status code.
9250 * @param pVCpu The cross context virtual CPU structure.
9251 * @param rcExit The reason for exiting to ring-3. Can be
9252 * VINF_VMM_UNKNOWN_RING3_CALL.
9253 */
9254static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
9255{
9256 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9257
9258 PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9259 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
9260 {
9261 VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
9262 pVCpu->hm.s.vmx.LastError.u32VmcsRev = *(uint32_t *)pVmcsInfo->pvVmcs;
9263 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
9264 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
9265 }
9266
9267 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
9268 VMMRZCallRing3Disable(pVCpu);
9269 Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
9270
9271 /*
9272 * Convert any pending HM events back to TRPM due to premature exits to ring-3.
9273 * We need to do this only on returns to ring-3 and not for longjmps to ring3.
9274 *
9275 * This is because execution may continue from ring-3 and we would need to inject
9276 * the event from there (hence place it back in TRPM).
9277 */
9278 if (pVCpu->hm.s.Event.fPending)
9279 {
9280 hmR0VmxPendingEventToTrpmTrap(pVCpu);
9281 Assert(!pVCpu->hm.s.Event.fPending);
9282
9283 /* Clear the events from the VMCS. */
9284 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
9285 AssertRCReturn(rc, rc);
9286 }
9287#ifdef VBOX_STRICT
9288 else
9289 {
9290 /*
9291 * Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
9292 * This can be pretty hard to debug otherwise, interrupts might get injected twice
9293 * occasionally, see @bugref{9180#c42}.
9294 *
9295 * However, if the VM-entry failed, any VM entry-interruption info. field would
9296 * be left unmodified as the event would not have been injected to the guest. In
9297 * such cases, don't assert, we're not going to continue guest execution anyway.
9298 */
9299 uint32_t uExitReason;
9300 uint32_t uEntryIntInfo;
9301 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
9302 rc |= VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
9303 AssertRC(rc);
9304 Assert(VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason) || !VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
9305 }
9306#endif
9307
9308 /*
9309 * Clear the interrupt-window and NMI-window VMCS controls as we could have got
9310 * a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
9311 * (e.g. TPR below threshold).
9312 */
9313 int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
9314 rc |= hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
9315 AssertRCReturn(rc, rc);
9316
9317 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
9318 and if we're injecting an event we should have a TRPM trap pending. */
9319 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
9320#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
9321 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
9322#endif
9323
9324 /* Save guest state and restore host state bits. */
9325 rc = hmR0VmxLeaveSession(pVCpu);
9326 AssertRCReturn(rc, rc);
9327 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
9328
9329 /* Thread-context hooks are unregistered at this point!!! */
9330
9331 /* Sync recompiler state. */
9332 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
9333 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
9334 | CPUM_CHANGED_LDTR
9335 | CPUM_CHANGED_GDTR
9336 | CPUM_CHANGED_IDTR
9337 | CPUM_CHANGED_TR
9338 | CPUM_CHANGED_HIDDEN_SEL_REGS);
9339 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
9340 && CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
9341 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
9342
9343 Assert(!pVCpu->hm.s.fClearTrapFlag);
9344
9345 /* Update the exit-to-ring 3 reason. */
9346 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
9347
9348 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
9349 if ( rcExit != VINF_EM_RAW_INTERRUPT
9350 || CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
9351 {
9352 Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
9353 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9354 }
9355
9356 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
9357
9358 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
9359 VMMRZCallRing3RemoveNotification(pVCpu);
9360 VMMRZCallRing3Enable(pVCpu);
9361
9362 return rc;
9363}
9364
9365
9366/**
9367 * VMMRZCallRing3() callback wrapper which saves the guest state before we
9368 * longjump to ring-3 and possibly get preempted.
9369 *
9370 * @returns VBox status code.
9371 * @param pVCpu The cross context virtual CPU structure.
9372 * @param enmOperation The operation causing the ring-3 longjump.
9373 * @param pvUser User argument, currently unused, NULL.
9374 */
9375static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
9376{
9377 RT_NOREF(pvUser);
9378 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
9379 {
9380 /*
9381 * !!! IMPORTANT !!!
9382 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
9383 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
9384 */
9385 VMMRZCallRing3RemoveNotification(pVCpu);
9386 VMMRZCallRing3Disable(pVCpu);
9387 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
9388 RTThreadPreemptDisable(&PreemptState);
9389
9390 PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9391 hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
9392 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
9393 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
9394
9395#if HC_ARCH_BITS == 64
9396 /* Restore host-state bits that VT-x only restores partially. */
9397 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
9398 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
9399 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
9400 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
9401#endif
9402
9403 /* Restore the lazy host MSRs as we're leaving VT-x context. */
9404 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
9405 hmR0VmxLazyRestoreHostMsrs(pVCpu);
9406
9407 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
9408 pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
9409 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
9410
9411 /* Clear the current VMCS data back to memory (shadow VMCS if any would have been
9412 cleared as part of importing the guest state above. */
9413 hmR0VmxClearVmcs(pVmcsInfo);
9414
9415 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
9416 VMMR0ThreadCtxHookDisable(pVCpu);
9417 HMR0LeaveCpu(pVCpu);
9418 RTThreadPreemptRestore(&PreemptState);
9419 return VINF_SUCCESS;
9420 }
9421
9422 Assert(pVCpu);
9423 Assert(pvUser);
9424 Assert(VMMRZCallRing3IsEnabled(pVCpu));
9425 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9426
9427 VMMRZCallRing3Disable(pVCpu);
9428 Assert(VMMR0IsLogFlushDisabled(pVCpu));
9429
9430 Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
9431
9432 int rc = hmR0VmxLongJmpToRing3(pVCpu);
9433 AssertRCReturn(rc, rc);
9434
9435 VMMRZCallRing3Enable(pVCpu);
9436 return VINF_SUCCESS;
9437}
9438
9439
9440/**
9441 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
9442 * stack.
9443 *
9444 * @returns Strict VBox status code (i.e. informational status codes too).
9445 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
9446 * @param pVCpu The cross context virtual CPU structure.
9447 * @param uValue The value to push to the guest stack.
9448 */
9449static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
9450{
9451 /*
9452 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
9453 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
9454 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
9455 */
9456 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9457 if (pCtx->sp == 1)
9458 return VINF_EM_RESET;
9459 pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
9460 int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
9461 AssertRC(rc);
9462 return rc;
9463}
9464
9465
9466/**
9467 * Injects an event into the guest upon VM-entry by updating the relevant fields
9468 * in the VM-entry area in the VMCS.
9469 *
9470 * @returns Strict VBox status code (i.e. informational status codes too).
9471 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
9472 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
9473 *
9474 * @param pVCpu The cross context virtual CPU structure.
9475 * @param pVmxTransient The VMX-transient structure.
9476 * @param pEvent The event being injected.
9477 * @param pfIntrState Pointer to the VT-x guest-interruptibility-state.
9478 * This will be updated if necessary. This cannot not
9479 * be NULL.
9480 * @param fStepping Whether we're single-stepping guest execution and
9481 * should return VINF_EM_DBG_STEPPED if the event is
9482 * injected directly (registers modified by us, not by
9483 * hardware on VM-entry).
9484 */
9485static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
9486 uint32_t *pfIntrState)
9487{
9488 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
9489 AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
9490 Assert(pfIntrState);
9491
9492 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9493 uint32_t u32IntInfo = pEvent->u64IntInfo;
9494 uint32_t const u32ErrCode = pEvent->u32ErrCode;
9495 uint32_t const cbInstr = pEvent->cbInstr;
9496 RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
9497 uint32_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
9498 uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
9499
9500#ifdef VBOX_STRICT
9501 /*
9502 * Validate the error-code-valid bit for hardware exceptions.
9503 * No error codes for exceptions in real-mode.
9504 *
9505 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
9506 */
9507 if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
9508 && !CPUMIsGuestInRealModeEx(pCtx))
9509 {
9510 switch (uVector)
9511 {
9512 case X86_XCPT_PF:
9513 case X86_XCPT_DF:
9514 case X86_XCPT_TS:
9515 case X86_XCPT_NP:
9516 case X86_XCPT_SS:
9517 case X86_XCPT_GP:
9518 case X86_XCPT_AC:
9519 AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
9520 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
9521 RT_FALL_THRU();
9522 default:
9523 break;
9524 }
9525 }
9526
9527 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
9528 Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
9529 || !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
9530#endif
9531
9532 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
9533
9534 /*
9535 * Hardware interrupts & exceptions cannot be delivered through the software interrupt
9536 * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
9537 * interrupt handler in the (real-mode) guest.
9538 *
9539 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
9540 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
9541 */
9542 if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
9543 {
9544 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
9545 {
9546 /*
9547 * For CPUs with unrestricted guest execution enabled and with the guest
9548 * in real-mode, we must not set the deliver-error-code bit.
9549 *
9550 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
9551 */
9552 u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
9553 }
9554 else
9555 {
9556 PVM pVM = pVCpu->CTX_SUFF(pVM);
9557 Assert(PDMVmmDevHeapIsEnabled(pVM));
9558 Assert(pVM->hm.s.vmx.pRealModeTSS);
9559 Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
9560
9561 /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
9562 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9563 int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_TABLE_MASK
9564 | CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_RFLAGS);
9565 AssertRCReturn(rc2, rc2);
9566
9567 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
9568 size_t const cbIdtEntry = sizeof(X86IDTR16);
9569 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
9570 {
9571 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
9572 if (uVector == X86_XCPT_DF)
9573 return VINF_EM_RESET;
9574
9575 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
9576 No error codes for exceptions in real-mode. */
9577 if (uVector == X86_XCPT_GP)
9578 {
9579 uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
9580 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
9581 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
9582 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
9583 HMEVENT EventXcptDf;
9584 RT_ZERO(EventXcptDf);
9585 EventXcptDf.u64IntInfo = uXcptDfInfo;
9586 return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
9587 }
9588
9589 /*
9590 * If we're injecting an event with no valid IDT entry, inject a #GP.
9591 * No error codes for exceptions in real-mode.
9592 *
9593 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
9594 */
9595 uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
9596 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
9597 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
9598 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
9599 HMEVENT EventXcptGp;
9600 RT_ZERO(EventXcptGp);
9601 EventXcptGp.u64IntInfo = uXcptGpInfo;
9602 return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
9603 }
9604
9605 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
9606 uint16_t uGuestIp = pCtx->ip;
9607 if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
9608 {
9609 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
9610 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
9611 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
9612 }
9613 else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
9614 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
9615
9616 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
9617 X86IDTR16 IdtEntry;
9618 RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
9619 rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
9620 AssertRCReturn(rc2, rc2);
9621
9622 /* Construct the stack frame for the interrupt/exception handler. */
9623 VBOXSTRICTRC rcStrict;
9624 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
9625 if (rcStrict == VINF_SUCCESS)
9626 {
9627 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
9628 if (rcStrict == VINF_SUCCESS)
9629 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
9630 }
9631
9632 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
9633 if (rcStrict == VINF_SUCCESS)
9634 {
9635 pCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
9636 pCtx->rip = IdtEntry.offSel;
9637 pCtx->cs.Sel = IdtEntry.uSel;
9638 pCtx->cs.ValidSel = IdtEntry.uSel;
9639 pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
9640 if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
9641 && uVector == X86_XCPT_PF)
9642 pCtx->cr2 = GCPtrFault;
9643
9644 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS | HM_CHANGED_GUEST_CR2
9645 | HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
9646 | HM_CHANGED_GUEST_RSP);
9647
9648 /*
9649 * If we delivered a hardware exception (other than an NMI) and if there was
9650 * block-by-STI in effect, we should clear it.
9651 */
9652 if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9653 {
9654 Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
9655 && uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
9656 Log4Func(("Clearing inhibition due to STI\n"));
9657 *pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
9658 }
9659
9660 Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
9661 u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
9662
9663 /*
9664 * The event has been truly dispatched to the guest. Mark it as no longer pending so
9665 * we don't attempt to undo it if we are returning to ring-3 before executing guest code.
9666 */
9667 pVCpu->hm.s.Event.fPending = false;
9668
9669 /* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
9670 if (fStepping)
9671 rcStrict = VINF_EM_DBG_STEPPED;
9672 }
9673 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
9674 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9675 return rcStrict;
9676 }
9677 }
9678
9679 /*
9680 * Validate.
9681 */
9682 Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
9683 Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
9684
9685 /*
9686 * Inject the event into the VMCS.
9687 */
9688 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
9689 if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
9690 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
9691 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
9692 AssertRCReturn(rc, rc);
9693
9694 /*
9695 * Update guest CR2 if this is a page-fault.
9696 */
9697 if ( VMX_ENTRY_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
9698 && uVector == X86_XCPT_PF)
9699 pCtx->cr2 = GCPtrFault;
9700
9701 Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
9702 return VINF_SUCCESS;
9703}
9704
9705
9706/**
9707 * Evaluates the event to be delivered to the guest and sets it as the pending
9708 * event.
9709 *
9710 * @returns Strict VBox status code (i.e. informational status codes too).
9711 * @param pVCpu The cross context virtual CPU structure.
9712 * @param pVmxTransient The VMX-transient structure.
9713 * @param pfIntrState Where to store the VT-x guest-interruptibility state.
9714 */
9715static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
9716{
9717 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9718 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9719
9720 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
9721 uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmxTransient);
9722 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
9723 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
9724 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
9725
9726 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
9727 Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
9728 Assert(!fBlockSti || pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
9729 Assert(!TRPMHasTrap(pVCpu));
9730 Assert(pfIntrState);
9731
9732 *pfIntrState = fIntrState;
9733
9734 /*
9735 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
9736 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
9737 */
9738 /** @todo SMI. SMIs take priority over NMIs. */
9739 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
9740 {
9741 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
9742 if ( !pVCpu->hm.s.Event.fPending
9743 && !fBlockNmi
9744 && !fBlockSti
9745 && !fBlockMovSS)
9746 {
9747#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9748 if ( pVmxTransient->fIsNestedGuest
9749 && CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
9750 return IEMExecVmxVmexitXcptNmi(pVCpu);
9751#endif
9752 hmR0VmxSetPendingXcptNmi(pVCpu);
9753 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
9754 Log4Func(("Pending NMI\n"));
9755 }
9756 else
9757 hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
9758 }
9759 /*
9760 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
9761 * a valid interrupt we -must- deliver the interrupt. We can no longer re-request it from the APIC.
9762 */
9763 else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)
9764 && !pVCpu->hm.s.fSingleInstruction)
9765 {
9766 Assert(!DBGFIsStepping(pVCpu));
9767 int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
9768 AssertRCReturn(rc, rc);
9769 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
9770 if ( !pVCpu->hm.s.Event.fPending
9771 && !fBlockInt
9772 && !fBlockSti
9773 && !fBlockMovSS)
9774 {
9775#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9776 if ( pVmxTransient->fIsNestedGuest
9777 && CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
9778 {
9779 VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0/* uVector */, true /* fIntPending */);
9780 if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
9781 return rcStrict;
9782 }
9783#endif
9784 uint8_t u8Interrupt;
9785 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
9786 if (RT_SUCCESS(rc))
9787 {
9788#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9789 if ( pVmxTransient->fIsNestedGuest
9790 && CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
9791 && CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
9792 {
9793 VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
9794 if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
9795 return rcStrict;
9796 }
9797#endif
9798 hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
9799 Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
9800 }
9801 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
9802 {
9803 if ( !pVmxTransient->fIsNestedGuest
9804 && (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
9805 hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
9806 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
9807
9808 /*
9809 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
9810 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
9811 * need to re-set this force-flag here.
9812 */
9813 }
9814 else
9815 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
9816 }
9817 else
9818 hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
9819 }
9820
9821 return VINF_SUCCESS;
9822}
9823
9824
9825/**
9826 * Injects any pending events into the guest if the guest is in a state to
9827 * receive them.
9828 *
9829 * @returns Strict VBox status code (i.e. informational status codes too).
9830 * @param pVCpu The cross context virtual CPU structure.
9831 * @param pVmxTransient The VMX-transient structure.
9832 * @param fIntrState The VT-x guest-interruptibility state.
9833 * @param fStepping Whether we are single-stepping the guest using the
9834 * hypervisor debugger and should return
9835 * VINF_EM_DBG_STEPPED if the event was dispatched
9836 * directly.
9837 */
9838static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
9839{
9840 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9841 Assert(VMMRZCallRing3IsEnabled(pVCpu));
9842
9843 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
9844 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
9845
9846 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
9847 Assert(!fBlockSti || pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
9848 Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
9849 Assert(!TRPMHasTrap(pVCpu));
9850
9851 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
9852 if (pVCpu->hm.s.Event.fPending)
9853 {
9854 /*
9855 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
9856 * pending even while injecting an event and in this case, we want a VM-exit as soon as
9857 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
9858 *
9859 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
9860 */
9861 uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
9862#ifdef VBOX_STRICT
9863 if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
9864 {
9865 bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
9866 Assert(!fBlockInt);
9867 Assert(!fBlockSti);
9868 Assert(!fBlockMovSS);
9869 }
9870 else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
9871 {
9872 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
9873 Assert(!fBlockSti);
9874 Assert(!fBlockMovSS);
9875 Assert(!fBlockNmi);
9876 }
9877#endif
9878 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
9879 uIntType));
9880
9881 /*
9882 * Inject the event and get any changes to the guest-interruptibility state.
9883 *
9884 * The guest-interruptibility state may need to be updated if we inject the event
9885 * into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
9886 */
9887 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
9888 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
9889
9890 /*
9891 * If we are executing a nested-guest make sure that we should intercept subsequent
9892 * events. The one we are injecting might be part of VM-entry.
9893 */
9894 if (pVmxTransient->fIsNestedGuest)
9895 pVCpu->cpum.GstCtx.hwvirt.vmx.fInterceptEvents = true;
9896
9897 if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
9898 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
9899 else
9900 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
9901 }
9902
9903 /*
9904 * Update the guest-interruptibility state.
9905 *
9906 * This is required for the real-on-v86 software interrupt injection case above, as well as
9907 * updates to the guest state from ring-3 or IEM/REM.
9908 */
9909 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
9910 AssertRCReturn(rc, rc);
9911
9912 /*
9913 * There's no need to clear the VM-entry interruption-information field here if we're not
9914 * injecting anything. VT-x clears the valid bit on every VM-exit.
9915 *
9916 * See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
9917 */
9918
9919 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
9920 NOREF(fBlockMovSS); NOREF(fBlockSti);
9921 return rcStrict;
9922}
9923
9924
9925/**
9926 * Enters the VT-x session.
9927 *
9928 * @returns VBox status code.
9929 * @param pVCpu The cross context virtual CPU structure.
9930 */
9931VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu)
9932{
9933 AssertPtr(pVCpu);
9934 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
9935 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9936
9937 LogFlowFunc(("pVCpu=%p\n", pVCpu));
9938 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9939 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9940
9941#ifdef VBOX_STRICT
9942 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
9943 RTCCUINTREG uHostCr4 = ASMGetCR4();
9944 if (!(uHostCr4 & X86_CR4_VMXE))
9945 {
9946 LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
9947 return VERR_VMX_X86_CR4_VMXE_CLEARED;
9948 }
9949#endif
9950
9951 /*
9952 * Load the appropriate VMCS as the current and active one.
9953 */
9954 PVMXVMCSINFO pVmcsInfo;
9955 bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
9956 if (!fInNestedGuestMode)
9957 pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
9958 else
9959 pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
9960 int rc = hmR0VmxLoadVmcs(pVmcsInfo);
9961 if (RT_SUCCESS(rc))
9962 {
9963 pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
9964 pVCpu->hm.s.fLeaveDone = false;
9965 Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9966
9967 /*
9968 * Do the EMT scheduled L1D flush here if needed.
9969 */
9970 if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9971 ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9972 else if (pVCpu->CTX_SUFF(pVM)->hm.s.fMdsClearOnSched)
9973 hmR0MdsClear();
9974 }
9975 return rc;
9976}
9977
9978
9979/**
9980 * The thread-context callback (only on platforms which support it).
9981 *
9982 * @param enmEvent The thread-context event.
9983 * @param pVCpu The cross context virtual CPU structure.
9984 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
9985 * @thread EMT(pVCpu)
9986 */
9987VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
9988{
9989 AssertPtr(pVCpu);
9990 RT_NOREF1(fGlobalInit);
9991
9992 switch (enmEvent)
9993 {
9994 case RTTHREADCTXEVENT_OUT:
9995 {
9996 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9997 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9998 VMCPU_ASSERT_EMT(pVCpu);
9999
10000 /* No longjmps (logger flushes, locks) in this fragile context. */
10001 VMMRZCallRing3Disable(pVCpu);
10002 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
10003
10004 /* Restore host-state (FPU, debug etc.) */
10005 if (!pVCpu->hm.s.fLeaveDone)
10006 {
10007 /*
10008 * Do -not- import the guest-state here as we might already be in the middle of importing
10009 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
10010 */
10011 hmR0VmxLeave(pVCpu, false /* fImportState */);
10012 pVCpu->hm.s.fLeaveDone = true;
10013 }
10014
10015 /* Leave HM context, takes care of local init (term). */
10016 int rc = HMR0LeaveCpu(pVCpu);
10017 AssertRC(rc);
10018
10019 /* Restore longjmp state. */
10020 VMMRZCallRing3Enable(pVCpu);
10021 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
10022 break;
10023 }
10024
10025 case RTTHREADCTXEVENT_IN:
10026 {
10027 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10028 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
10029 VMCPU_ASSERT_EMT(pVCpu);
10030
10031 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
10032 VMMRZCallRing3Disable(pVCpu);
10033 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
10034
10035 /* Initialize the bare minimum state required for HM. This takes care of
10036 initializing VT-x if necessary (onlined CPUs, local init etc.) */
10037 int rc = hmR0EnterCpu(pVCpu);
10038 AssertRC(rc);
10039 Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
10040 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
10041
10042 /* Load the active VMCS as the current one. */
10043 PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10044 rc = hmR0VmxLoadVmcs(pVmcsInfo);
10045 AssertRC(rc);
10046 Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
10047 pVCpu->hm.s.fLeaveDone = false;
10048
10049 /* Do the EMT scheduled L1D flush if needed. */
10050 if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
10051 ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
10052
10053 /* Restore longjmp state. */
10054 VMMRZCallRing3Enable(pVCpu);
10055 break;
10056 }
10057
10058 default:
10059 break;
10060 }
10061}
10062
10063
10064/**
10065 * Exports the host state into the VMCS host-state area.
10066 * Sets up the VM-exit MSR-load area.
10067 *
10068 * The CPU state will be loaded from these fields on every successful VM-exit.
10069 *
10070 * @returns VBox status code.
10071 * @param pVCpu The cross context virtual CPU structure.
10072 *
10073 * @remarks No-long-jump zone!!!
10074 */
10075static int hmR0VmxExportHostState(PVMCPU pVCpu)
10076{
10077 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10078
10079 int rc = VINF_SUCCESS;
10080 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
10081 {
10082 rc = hmR0VmxExportHostControlRegs();
10083 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10084
10085 rc = hmR0VmxExportHostSegmentRegs(pVCpu);
10086 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10087
10088 rc = hmR0VmxExportHostMsrs(pVCpu);
10089 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10090
10091 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
10092 }
10093 return rc;
10094}
10095
10096
10097/**
10098 * Saves the host state in the VMCS host-state.
10099 *
10100 * @returns VBox status code.
10101 * @param pVCpu The cross context virtual CPU structure.
10102 *
10103 * @remarks No-long-jump zone!!!
10104 */
10105VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
10106{
10107 AssertPtr(pVCpu);
10108 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10109
10110 /*
10111 * Export the host state here while entering HM context.
10112 * When thread-context hooks are used, we might get preempted and have to re-save the host
10113 * state but most of the time we won't be, so do it here before we disable interrupts.
10114 */
10115 return hmR0VmxExportHostState(pVCpu);
10116}
10117
10118
10119/**
10120 * Exports the guest state into the VMCS guest-state area.
10121 *
10122 * The will typically be done before VM-entry when the guest-CPU state and the
10123 * VMCS state may potentially be out of sync.
10124 *
10125 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
10126 * VM-entry controls.
10127 * Sets up the appropriate VMX non-root function to execute guest code based on
10128 * the guest CPU mode.
10129 *
10130 * @returns VBox strict status code.
10131 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
10132 * without unrestricted guest execution and the VMMDev is not presently
10133 * mapped (e.g. EFI32).
10134 *
10135 * @param pVCpu The cross context virtual CPU structure.
10136 * @param pVmxTransient The VMX-transient structure.
10137 *
10138 * @remarks No-long-jump zone!!!
10139 */
10140static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10141{
10142 AssertPtr(pVCpu);
10143 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
10144 LogFlowFunc(("pVCpu=%p\n", pVCpu));
10145
10146 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
10147
10148 /*
10149 * Determine real-on-v86 mode.
10150 * Used when the guest is in real-mode and unrestricted guest execution is not used.
10151 */
10152 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10153 if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
10154 || !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
10155 pVmcsInfo->RealMode. fRealOnV86Active = false;
10156 else
10157 {
10158 Assert(!pVmxTransient->fIsNestedGuest);
10159 pVmcsInfo->RealMode.fRealOnV86Active = true;
10160 }
10161
10162 /*
10163 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
10164 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
10165 */
10166 /** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
10167 * hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
10168 * be a need to evaluate this everytime since I'm pretty sure we intercept
10169 * all guest paging mode changes. */
10170 int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
10171 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10172
10173 rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
10174 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10175
10176 rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
10177 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10178
10179 VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
10180 if (rcStrict == VINF_SUCCESS)
10181 { /* likely */ }
10182 else
10183 {
10184 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
10185 return rcStrict;
10186 }
10187
10188 rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
10189 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10190
10191 rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
10192 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10193
10194 rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
10195 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10196
10197 rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
10198 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10199
10200 rc = hmR0VmxExportGuestRip(pVCpu);
10201 rc |= hmR0VmxExportGuestRsp(pVCpu);
10202 rc |= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
10203 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10204
10205 rc = hmR0VmxExportGuestHwvirtState(pVCpu, pVmxTransient);
10206 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10207
10208 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
10209 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
10210 | HM_CHANGED_GUEST_CR2
10211 | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
10212 | HM_CHANGED_GUEST_X87
10213 | HM_CHANGED_GUEST_SSE_AVX
10214 | HM_CHANGED_GUEST_OTHER_XSAVE
10215 | HM_CHANGED_GUEST_XCRx
10216 | HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
10217 | HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
10218 | HM_CHANGED_GUEST_TSC_AUX
10219 | HM_CHANGED_GUEST_OTHER_MSRS
10220 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
10221
10222 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
10223 return rc;
10224}
10225
10226
10227/**
10228 * Exports the state shared between the host and guest into the VMCS.
10229 *
10230 * @param pVCpu The cross context virtual CPU structure.
10231 * @param pVmxTransient The VMX-transient structure.
10232 *
10233 * @remarks No-long-jump zone!!!
10234 */
10235static void hmR0VmxExportSharedState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10236{
10237 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10238 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10239
10240 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
10241 {
10242 int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
10243 AssertRC(rc);
10244 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
10245
10246 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
10247 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
10248 {
10249 rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
10250 AssertRC(rc);
10251 }
10252 }
10253
10254 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
10255 {
10256 hmR0VmxLazyLoadGuestMsrs(pVCpu);
10257 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
10258 }
10259
10260 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
10261 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10262}
10263
10264
10265/**
10266 * Worker for loading the guest-state bits in the inner VT-x execution loop.
10267 *
10268 * @returns Strict VBox status code (i.e. informational status codes too).
10269 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
10270 * without unrestricted guest execution and the VMMDev is not presently
10271 * mapped (e.g. EFI32).
10272 *
10273 * @param pVCpu The cross context virtual CPU structure.
10274 * @param pVmxTransient The VMX-transient structure.
10275 *
10276 * @remarks No-long-jump zone!!!
10277 */
10278static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10279{
10280 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
10281 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10282 Assert(VMMR0IsLogFlushDisabled(pVCpu));
10283
10284#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
10285 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
10286#endif
10287
10288 /*
10289 * For many exits it's only RIP that changes and hence try to export it first
10290 * without going through a lot of change flag checks.
10291 */
10292 VBOXSTRICTRC rcStrict;
10293 uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
10294 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
10295 if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
10296 {
10297 rcStrict = hmR0VmxExportGuestRip(pVCpu);
10298 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10299 { /* likely */}
10300 else
10301 AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
10302 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
10303 }
10304 else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
10305 {
10306 rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
10307 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10308 { /* likely */}
10309 else
10310 {
10311 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
10312 VBOXSTRICTRC_VAL(rcStrict)));
10313 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10314 return rcStrict;
10315 }
10316 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
10317 }
10318 else
10319 rcStrict = VINF_SUCCESS;
10320
10321#ifdef VBOX_STRICT
10322 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
10323 fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
10324 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
10325 AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
10326 ("fCtxChanged=%#RX64\n", fCtxChanged));
10327#endif
10328 return rcStrict;
10329}
10330
10331
10332/**
10333 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10334 * and update error record fields accordingly.
10335 *
10336 * @returns VMX_IGS_* error codes.
10337 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10338 * wrong with the guest state.
10339 *
10340 * @param pVCpu The cross context virtual CPU structure.
10341 * @param pVmcsInfo The VMCS info. object.
10342 *
10343 * @remarks This function assumes our cache of the VMCS controls
10344 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10345 */
10346static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
10347{
10348#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10349#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10350 uError = (err); \
10351 break; \
10352 } else do { } while (0)
10353
10354 int rc;
10355 PVM pVM = pVCpu->CTX_SUFF(pVM);
10356 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10357 uint32_t uError = VMX_IGS_ERROR;
10358 uint32_t u32Val;
10359 bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10360
10361 do
10362 {
10363 /*
10364 * CR0.
10365 */
10366 /** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
10367 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10368 uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10369 /* Exceptions for unrestricted guest execution for fixed CR0 bits (PE, PG).
10370 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10371 if (fUnrestrictedGuest)
10372 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
10373
10374 uint32_t u32GuestCr0;
10375 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10376 AssertRCBreak(rc);
10377 HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10378 HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10379 if ( !fUnrestrictedGuest
10380 && (u32GuestCr0 & X86_CR0_PG)
10381 && !(u32GuestCr0 & X86_CR0_PE))
10382 {
10383 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10384 }
10385
10386 /*
10387 * CR4.
10388 */
10389 /** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
10390 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10391 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10392
10393 uint32_t u32GuestCr4;
10394 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10395 AssertRCBreak(rc);
10396 HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10397 HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10398
10399 /*
10400 * IA32_DEBUGCTL MSR.
10401 */
10402 uint64_t u64Val;
10403 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10404 AssertRCBreak(rc);
10405 if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10406 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10407 {
10408 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10409 }
10410 uint64_t u64DebugCtlMsr = u64Val;
10411
10412#ifdef VBOX_STRICT
10413 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10414 AssertRCBreak(rc);
10415 Assert(u32Val == pVmcsInfo->u32EntryCtls);
10416#endif
10417 bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
10418
10419 /*
10420 * RIP and RFLAGS.
10421 */
10422 uint32_t u32Eflags;
10423#if HC_ARCH_BITS == 64
10424 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10425 AssertRCBreak(rc);
10426 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10427 if ( !fLongModeGuest
10428 || !pCtx->cs.Attr.n.u1Long)
10429 {
10430 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10431 }
10432 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10433 * must be identical if the "IA-32e mode guest" VM-entry
10434 * control is 1 and CS.L is 1. No check applies if the
10435 * CPU supports 64 linear-address bits. */
10436
10437 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10438 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10439 AssertRCBreak(rc);
10440 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10441 VMX_IGS_RFLAGS_RESERVED);
10442 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10443 u32Eflags = u64Val;
10444#else
10445 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10446 AssertRCBreak(rc);
10447 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10448 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10449#endif
10450
10451 if ( fLongModeGuest
10452 || ( fUnrestrictedGuest
10453 && !(u32GuestCr0 & X86_CR0_PE)))
10454 {
10455 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10456 }
10457
10458 uint32_t u32EntryInfo;
10459 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10460 AssertRCBreak(rc);
10461 if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
10462 && VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
10463 {
10464 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10465 }
10466
10467 /*
10468 * 64-bit checks.
10469 */
10470#if HC_ARCH_BITS == 64
10471 if (fLongModeGuest)
10472 {
10473 HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10474 HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10475 }
10476
10477 if ( !fLongModeGuest
10478 && (u32GuestCr4 & X86_CR4_PCIDE))
10479 {
10480 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10481 }
10482
10483 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10484 * 51:32 beyond the processor's physical-address width are 0. */
10485
10486 if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10487 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10488 {
10489 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10490 }
10491
10492 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10493 AssertRCBreak(rc);
10494 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10495
10496 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10497 AssertRCBreak(rc);
10498 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10499#endif
10500
10501 /*
10502 * PERF_GLOBAL MSR.
10503 */
10504 if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
10505 {
10506 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10507 AssertRCBreak(rc);
10508 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10509 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10510 }
10511
10512 /*
10513 * PAT MSR.
10514 */
10515 if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
10516 {
10517 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10518 AssertRCBreak(rc);
10519 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10520 for (unsigned i = 0; i < 8; i++)
10521 {
10522 uint8_t u8Val = (u64Val & 0xff);
10523 if ( u8Val != 0 /* UC */
10524 && u8Val != 1 /* WC */
10525 && u8Val != 4 /* WT */
10526 && u8Val != 5 /* WP */
10527 && u8Val != 6 /* WB */
10528 && u8Val != 7 /* UC- */)
10529 {
10530 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10531 }
10532 u64Val >>= 8;
10533 }
10534 }
10535
10536 /*
10537 * EFER MSR.
10538 */
10539 if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
10540 {
10541 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10542 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10543 AssertRCBreak(rc);
10544 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10545 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10546 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
10547 & VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
10548 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10549 /** @todo r=ramshankar: Unrestricted check here is probably wrong, see
10550 * iemVmxVmentryCheckGuestState(). */
10551 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10552 || !(u32GuestCr0 & X86_CR0_PG)
10553 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10554 VMX_IGS_EFER_LMA_LME_MISMATCH);
10555 }
10556
10557 /*
10558 * Segment registers.
10559 */
10560 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10561 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10562 if (!(u32Eflags & X86_EFL_VM))
10563 {
10564 /* CS */
10565 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10566 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10567 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10568 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10569 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10570 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10571 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10572 /* CS cannot be loaded with NULL in protected mode. */
10573 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10574 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10575 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10576 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10577 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10578 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10579 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10580 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10581 else
10582 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10583
10584 /* SS */
10585 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10586 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10587 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10588 if ( !(pCtx->cr0 & X86_CR0_PE)
10589 || pCtx->cs.Attr.n.u4Type == 3)
10590 {
10591 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10592 }
10593 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10594 {
10595 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10596 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10597 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10598 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10599 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10600 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10601 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10602 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10603 }
10604
10605 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
10606 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10607 {
10608 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10609 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10610 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10611 || pCtx->ds.Attr.n.u4Type > 11
10612 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10613 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10614 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10615 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10616 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10617 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10618 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10619 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10620 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10621 }
10622 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10623 {
10624 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10625 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10626 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10627 || pCtx->es.Attr.n.u4Type > 11
10628 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10629 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10630 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10631 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10632 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10633 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10634 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10635 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10636 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10637 }
10638 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10639 {
10640 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10641 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10642 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10643 || pCtx->fs.Attr.n.u4Type > 11
10644 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10645 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10646 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10647 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10648 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10649 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10650 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10651 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10652 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10653 }
10654 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10655 {
10656 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10657 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10658 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10659 || pCtx->gs.Attr.n.u4Type > 11
10660 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10661 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10662 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10663 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10664 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10665 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10666 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10667 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10668 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10669 }
10670 /* 64-bit capable CPUs. */
10671#if HC_ARCH_BITS == 64
10672 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10673 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10674 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10675 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10676 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10677 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10678 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10679 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10680 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10681 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10682 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10683#endif
10684 }
10685 else
10686 {
10687 /* V86 mode checks. */
10688 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10689 if (pVmcsInfo->RealMode.fRealOnV86Active)
10690 {
10691 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10692 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10693 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10694 }
10695 else
10696 {
10697 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10698 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10699 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10700 }
10701
10702 /* CS */
10703 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10704 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10705 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10706 /* SS */
10707 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10708 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10709 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10710 /* DS */
10711 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10712 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10713 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10714 /* ES */
10715 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10716 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10717 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10718 /* FS */
10719 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10720 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10721 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10722 /* GS */
10723 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10724 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10725 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10726 /* 64-bit capable CPUs. */
10727#if HC_ARCH_BITS == 64
10728 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10729 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10730 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10731 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10732 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10733 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10734 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10735 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10736 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10737 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10738 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10739#endif
10740 }
10741
10742 /*
10743 * TR.
10744 */
10745 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10746 /* 64-bit capable CPUs. */
10747#if HC_ARCH_BITS == 64
10748 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10749#endif
10750 if (fLongModeGuest)
10751 {
10752 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10753 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10754 }
10755 else
10756 {
10757 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10758 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10759 VMX_IGS_TR_ATTR_TYPE_INVALID);
10760 }
10761 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10762 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10763 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10764 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10765 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10766 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10767 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10768 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10769
10770 /*
10771 * GDTR and IDTR.
10772 */
10773#if HC_ARCH_BITS == 64
10774 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10775 AssertRCBreak(rc);
10776 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10777
10778 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10779 AssertRCBreak(rc);
10780 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10781#endif
10782
10783 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10784 AssertRCBreak(rc);
10785 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10786
10787 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10788 AssertRCBreak(rc);
10789 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10790
10791 /*
10792 * Guest Non-Register State.
10793 */
10794 /* Activity State. */
10795 uint32_t u32ActivityState;
10796 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10797 AssertRCBreak(rc);
10798 HMVMX_CHECK_BREAK( !u32ActivityState
10799 || (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
10800 VMX_IGS_ACTIVITY_STATE_INVALID);
10801 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10802 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10803 uint32_t u32IntrState;
10804 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
10805 AssertRCBreak(rc);
10806 if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
10807 || u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10808 {
10809 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10810 }
10811
10812 /** @todo Activity state and injecting interrupts. Left as a todo since we
10813 * currently don't use activity states but ACTIVE. */
10814
10815 HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10816 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10817
10818 /* Guest interruptibility-state. */
10819 HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10820 HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
10821 != (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10822 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10823 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10824 || !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10825 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10826 if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
10827 {
10828 if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
10829 {
10830 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10831 && !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10832 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10833 }
10834 else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
10835 {
10836 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10837 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10838 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10839 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10840 }
10841 }
10842 /** @todo Assumes the processor is not in SMM. */
10843 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10844 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10845 HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10846 || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10847 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10848 if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
10849 && VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
10850 && VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
10851 {
10852 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
10853 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10854 }
10855
10856 /* Pending debug exceptions. */
10857#if HC_ARCH_BITS == 64
10858 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
10859 AssertRCBreak(rc);
10860 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10861 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10862 u32Val = u64Val; /* For pending debug exceptions checks below. */
10863#else
10864 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
10865 AssertRCBreak(rc);
10866 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10867 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10868#endif
10869
10870 if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10871 || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
10872 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10873 {
10874 if ( (u32Eflags & X86_EFL_TF)
10875 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10876 {
10877 /* Bit 14 is PendingDebug.BS. */
10878 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10879 }
10880 if ( !(u32Eflags & X86_EFL_TF)
10881 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10882 {
10883 /* Bit 14 is PendingDebug.BS. */
10884 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10885 }
10886 }
10887
10888 /* VMCS link pointer. */
10889 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10890 AssertRCBreak(rc);
10891 if (u64Val != UINT64_C(0xffffffffffffffff))
10892 {
10893 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10894 /** @todo Bits beyond the processor's physical-address width MBZ. */
10895 /** @todo SMM checks. */
10896 Assert(pVmcsInfo->HCPhysShadowVmcs == u64Val);
10897 Assert(pVmcsInfo->pvShadowVmcs);
10898 VMXVMCSREVID VmcsRevId;
10899 VmcsRevId.u = *(uint32_t *)pVmcsInfo->pvShadowVmcs;
10900 HMVMX_CHECK_BREAK(VmcsRevId.n.u31RevisionId == RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID),
10901 VMX_IGS_VMCS_LINK_PTR_SHADOW_VMCS_ID_INVALID);
10902 HMVMX_CHECK_BREAK(VmcsRevId.n.fIsShadowVmcs == (uint32_t)!!(pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING),
10903 VMX_IGS_VMCS_LINK_PTR_NOT_SHADOW);
10904 }
10905
10906 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10907 * not using nested paging? */
10908 if ( pVM->hm.s.fNestedPaging
10909 && !fLongModeGuest
10910 && CPUMIsGuestInPAEModeEx(pCtx))
10911 {
10912 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10913 AssertRCBreak(rc);
10914 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10915
10916 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10917 AssertRCBreak(rc);
10918 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10919
10920 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10921 AssertRCBreak(rc);
10922 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10923
10924 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10925 AssertRCBreak(rc);
10926 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10927 }
10928
10929 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10930 if (uError == VMX_IGS_ERROR)
10931 uError = VMX_IGS_REASON_NOT_FOUND;
10932 } while (0);
10933
10934 pVCpu->hm.s.u32HMError = uError;
10935 return uError;
10936
10937#undef HMVMX_ERROR_BREAK
10938#undef HMVMX_CHECK_BREAK
10939}
10940
10941
10942/**
10943 * Setup the APIC-access page for virtualizing APIC access.
10944 *
10945 * This can cause a longjumps to R3 due to the acquisition of the PGM lock, hence
10946 * this not done as part of exporting guest state, see @bugref{8721}.
10947 *
10948 * @returns VBox status code.
10949 * @param pVCpu The cross context virtual CPU structure.
10950 */
10951static int hmR0VmxMapHCApicAccessPage(PVMCPU pVCpu)
10952{
10953 PVM pVM = pVCpu->CTX_SUFF(pVM);
10954 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
10955
10956 Assert(PDMHasApic(pVM));
10957 Assert(u64MsrApicBase);
10958
10959 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
10960 Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
10961
10962 /* Unalias any existing mapping. */
10963 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
10964 AssertRCReturn(rc, rc);
10965
10966 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
10967 Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
10968 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
10969 AssertRCReturn(rc, rc);
10970
10971 /* Update the per-VCPU cache of the APIC base MSR. */
10972 pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
10973 return VINF_SUCCESS;
10974}
10975
10976
10977#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10978/**
10979 * Merges the guest with the nested-guest MSR bitmap in preparation of executing the
10980 * nested-guest using hardware-assisted VMX.
10981 *
10982 * @param pVCpu The cross context virtual CPU structure.
10983 * @param pVmcsInfoNstGst The nested-guest VMCS info. object.
10984 * @param pVmcsInfoGst The guest VMCS info. object.
10985 */
10986static void hmR0VmxMergeMsrBitmapNested(PCVMCPU pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
10987{
10988 uint32_t const cbMsrBitmap = X86_PAGE_4K_SIZE;
10989 uint64_t *pu64MsrBitmap = (uint64_t *)pVmcsInfoNstGst->pvMsrBitmap;
10990 Assert(pu64MsrBitmap);
10991
10992 /*
10993 * We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
10994 * MSR that is intercepted by the guest is also intercepted while executing the
10995 * nested-guest using hardware-assisted VMX.
10996 *
10997 * Note! If the nested-guest is not using an MSR bitmap, ever MSR must cause a
10998 * nested-guest VM-exit even if the outer guest is not intercepting some
10999 * MSRs. We cannot assume the caller has initialized the nested-guest
11000 * MSR bitmap in this case.
11001 *
11002 * The guest hypervisor may also switch whether it uses MSR bitmaps for
11003 * each VM-entry, hence initializing it once per-VM while setting up the
11004 * nested-guest VMCS is not sufficient.
11005 */
11006 PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
11007 if (pVmcsNstGst->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
11008 {
11009 uint64_t const *pu64MsrBitmapNstGst = (uint64_t const *)pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
11010 uint64_t const *pu64MsrBitmapGst = (uint64_t const *)pVmcsInfoGst->pvMsrBitmap;
11011 Assert(pu64MsrBitmapNstGst);
11012 Assert(pu64MsrBitmapGst);
11013
11014 uint32_t const cFrags = cbMsrBitmap / sizeof(uint64_t);
11015 for (uint32_t i = 0; i < cFrags; i++)
11016 pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] | pu64MsrBitmapGst[i];
11017 }
11018 else
11019 ASMMemFill32(pu64MsrBitmap, cbMsrBitmap, UINT32_C(0xffffffff));
11020}
11021
11022
11023/**
11024 * Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
11025 * hardware-assisted VMX execution of the nested-guest.
11026 *
11027 * For a guest, we don't modify these controls once we set up the VMCS and hence
11028 * this function is never called.
11029 *
11030 * For nested-guests since the guest hypervisor provides these controls on every
11031 * nested-guest VM-entry and could potentially change them everytime we need to
11032 * merge them before every nested-guest VM-entry.
11033 *
11034 * @returns VBox status code.
11035 * @param pVCpu The cross context virtual CPU structure.
11036 */
11037static int hmR0VmxMergeVmcsNested(PVMCPU pVCpu)
11038{
11039 PVM pVM = pVCpu->CTX_SUFF(pVM);
11040 PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
11041 PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
11042 Assert(pVmcsNstGst);
11043
11044 /*
11045 * Merge the controls with the requirements of the guest VMCS.
11046 *
11047 * We do not need to validate the nested-guest VMX features specified in the
11048 * nested-guest VMCS with the features supported by the physical CPU as it's
11049 * already done by the VMLAUNCH/VMRESUME instruction emulation.
11050 *
11051 * This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the
11052 * guest are derived from the VMX features supported by the physical CPU.
11053 */
11054
11055 /* Pin-based VM-execution controls. */
11056 uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls | pVmcsInfoGst->u32PinCtls;
11057
11058 /* Processor-based VM-execution controls. */
11059 uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
11060 | (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
11061 | VMX_PROC_CTLS_NMI_WINDOW_EXIT
11062 | VMX_PROC_CTLS_USE_TPR_SHADOW
11063 | VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
11064
11065 /* Secondary processor-based VM-execution controls. */
11066 uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
11067 | (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
11068 | VMX_PROC_CTLS2_INVPCID
11069 | VMX_PROC_CTLS2_RDTSCP
11070 | VMX_PROC_CTLS2_XSAVES_XRSTORS
11071 | VMX_PROC_CTLS2_APIC_REG_VIRT
11072 | VMX_PROC_CTLS2_VIRT_INT_DELIVERY
11073 | VMX_PROC_CTLS2_VMFUNC));
11074
11075 /*
11076 * VM-entry controls:
11077 * These controls contains state that depends on the nested-guest state (primarily
11078 * EFER MSR) and is thus not constant between VMLAUNCH/VMRESUME and the nested-guest
11079 * VM-exit. Although the guest hypervisor cannot change it, we need to in order to
11080 * properly continue executing the nested-guest if the EFER MSR changes but does not
11081 * cause a nested-guest VM-exits.
11082 *
11083 * VM-exit controls:
11084 * These controls specify the host state on return. We cannot use the controls from
11085 * the guest hypervisor state as is as it would contain the guest state rather than
11086 * the host state. Since the host state is subject to change (e.g. preemption, trips
11087 * to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
11088 * through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
11089 *
11090 * VM-entry MSR-load:
11091 * The guest MSRs from the VM-entry MSR-load area are already loaded into the
11092 * guest-CPU context by the VMLAUNCH/VMRESUME instruction emulation.
11093 *
11094 * VM-exit MSR-store:
11095 * The VM-exit emulation will take care of populating the MSRs from the guest-CPU
11096 * context back into the VM-exit MSR-store area.
11097 *
11098 * VM-exit MSR-load areas:
11099 * This must contain the real host MSRs with hardware-assisted VMX execution. Hence,
11100 * we can entirely ignore what the guest hypervisor wants to load here.
11101 */
11102
11103 /*
11104 * Exception bitmap.
11105 *
11106 * We could remove #UD from the guest bitmap and merge it with the nested-guest
11107 * bitmap here (and avoid doing anything while exporting nested-guest state), but to
11108 * keep the code more flexible if intercepting exceptions become more dynamic in
11109 * the future we do it as part of exporting the nested-guest state.
11110 */
11111 uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap | pVmcsInfoGst->u32XcptBitmap;
11112
11113 /*
11114 * CR0/CR4 guest/host mask.
11115 *
11116 * Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest
11117 * must cause VM-exits, so we need to merge them here.
11118 */
11119 uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u | pVmcsInfoGst->u64Cr0Mask;
11120 uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u | pVmcsInfoGst->u64Cr4Mask;
11121
11122 /*
11123 * Page-fault error-code mask and match.
11124 *
11125 * Although we require unrestricted guest execution (and thereby nested-paging) for
11126 * hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
11127 * normally intercept #PFs, it might intercept them for debugging purposes.
11128 *
11129 * If the outer guest is not intercepting #PFs, we can use the nested-guest #PF
11130 * filters. If the outer guest is intercepting #PFs we must intercept all #PFs.
11131 */
11132 uint32_t u32XcptPFMask;
11133 uint32_t u32XcptPFMatch;
11134 if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
11135 {
11136 u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
11137 u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
11138 }
11139 else
11140 {
11141 u32XcptPFMask = 0;
11142 u32XcptPFMatch = 0;
11143 }
11144
11145 /*
11146 * Pause-Loop exiting.
11147 */
11148 uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
11149 uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
11150
11151 /*
11152 * I/O Bitmap.
11153 *
11154 * We do not use the I/O bitmap that may be provided by the guest hypervisor as we
11155 * always intercept all I/O port accesses.
11156 */
11157 Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
11158
11159 /*
11160 * APIC-access page.
11161 *
11162 * The APIC-access page address has already been initialized while setting up the
11163 * nested-guest VMCS. In theory, even if the guest-physical address is invalid, it
11164 * should not be on any consequence to the host or to the guest for that matter, but
11165 * we only accept valid addresses verified by the VMLAUNCH/VMRESUME instruction
11166 * emulation to keep it simple.
11167 */
11168
11169 /*
11170 * Virtual-APIC page and TPR threshold.
11171 *
11172 * We shall use the host-physical address of the virtual-APIC page in guest memory directly.
11173 * For this reason, we can access the virtual-APIC page of the nested-guest only using
11174 * PGM physical handlers as we must not assume a kernel virtual-address mapping exists and
11175 * requesting PGM for a mapping could be expensive/resource intensive (PGM mapping cache).
11176 */
11177 RTHCPHYS HCPhysVirtApic = NIL_RTHCPHYS;
11178 uint32_t const u32TprThreshold = pVmcsNstGst->u32TprThreshold;
11179 if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
11180 {
11181 int rc = PGMPhysGCPhys2HCPhys(pVM, pVmcsNstGst->u64AddrVirtApic.u, &HCPhysVirtApic);
11182
11183 /*
11184 * If the guest hypervisor has loaded crap into the virtual-APIC page field
11185 * we would fail to obtain a valid host-physical address for its guest-physical
11186 * address.
11187 *
11188 * We currently do not support this scenario. Maybe in the future if there is a
11189 * pressing need we can explore making this particular set of conditions work.
11190 * Right now we just cause a VM-entry failure.
11191 *
11192 * This has already been checked by VMLAUNCH/VMRESUME instruction emulation,
11193 * so should not really failure at the moment.
11194 */
11195 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
11196 }
11197 else
11198 {
11199 /*
11200 * We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
11201 * used by the guest hypervisor. Preventing MMIO accesses to the physical APIC will
11202 * be taken care of by EPT/shadow paging.
11203 */
11204 if (pVM->hm.s.fAllow64BitGuests)
11205 {
11206 u32ProcCtls |= VMX_PROC_CTLS_CR8_STORE_EXIT
11207 | VMX_PROC_CTLS_CR8_LOAD_EXIT;
11208 }
11209 }
11210
11211 /*
11212 * Validate basic assumptions.
11213 */
11214 PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
11215 Assert(pVM->hm.s.vmx.fAllowUnrestricted);
11216 Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
11217 Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
11218
11219 /*
11220 * Commit it to the nested-guest VMCS.
11221 */
11222 int rc = VINF_SUCCESS;
11223 if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
11224 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
11225 if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
11226 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
11227 if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
11228 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
11229 if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
11230 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
11231 if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
11232 rc |= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
11233 if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
11234 rc |= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
11235 if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
11236 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
11237 if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
11238 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
11239 if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
11240 && (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
11241 {
11242 Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
11243 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
11244 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
11245 }
11246 if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
11247 {
11248 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
11249 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
11250 }
11251 AssertRCReturn(rc, rc);
11252
11253 /*
11254 * Update the nested-guest VMCS cache.
11255 */
11256 pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
11257 pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
11258 pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
11259 pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
11260 pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
11261 pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
11262 pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
11263 pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
11264 pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
11265
11266 /*
11267 * MSR bitmap.
11268 *
11269 * The MSR bitmap address has already been initialized while setting up the
11270 * nested-guest VMCS, here we need to merge the MSR bitmaps.
11271 */
11272 if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
11273 hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
11274
11275 return VINF_SUCCESS;
11276}
11277#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
11278
11279
11280/**
11281 * Does the preparations before executing guest code in VT-x.
11282 *
11283 * This may cause longjmps to ring-3 and may even result in rescheduling to the
11284 * recompiler/IEM. We must be cautious what we do here regarding committing
11285 * guest-state information into the VMCS assuming we assuredly execute the
11286 * guest in VT-x mode.
11287 *
11288 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
11289 * the common-state (TRPM/forceflags), we must undo those changes so that the
11290 * recompiler/IEM can (and should) use them when it resumes guest execution.
11291 * Otherwise such operations must be done when we can no longer exit to ring-3.
11292 *
11293 * @returns Strict VBox status code (i.e. informational status codes too).
11294 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
11295 * have been disabled.
11296 * @retval VINF_VMX_VMEXIT if a nested-guest VM-exit occurs (e.g., while evaluating
11297 * pending events).
11298 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
11299 * double-fault into the guest.
11300 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
11301 * dispatched directly.
11302 * @retval VINF_* scheduling changes, we have to go back to ring-3.
11303 *
11304 * @param pVCpu The cross context virtual CPU structure.
11305 * @param pVmxTransient The VMX-transient structure.
11306 * @param fStepping Whether we are single-stepping the guest in the
11307 * hypervisor debugger. Makes us ignore some of the reasons
11308 * for returning to ring-3, and return VINF_EM_DBG_STEPPED
11309 * if event dispatching took place.
11310 */
11311static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
11312{
11313 Assert(VMMRZCallRing3IsEnabled(pVCpu));
11314
11315#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
11316 if (pVmxTransient->fIsNestedGuest)
11317 {
11318 RT_NOREF2(pVCpu, fStepping);
11319 Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
11320 return VINF_EM_RESCHEDULE_REM;
11321 }
11322#endif
11323
11324#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
11325 PGMRZDynMapFlushAutoSet(pVCpu);
11326#endif
11327
11328 /*
11329 * Check and process force flag actions, some of which might require us to go back to ring-3.
11330 */
11331 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
11332 if (rcStrict == VINF_SUCCESS)
11333 { /* FFs don't get set all the time. */ }
11334 else
11335 return rcStrict;
11336
11337#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11338 /*
11339 * Switch to the nested-guest VMCS as we may have transitioned into executing
11340 * the nested-guest without leaving ring-0. Otherwise, if we came from ring-3
11341 * we would load the nested-guest VMCS while entering the VMX ring-0 session.
11342 *
11343 * We do this as late as possible to minimize (though not completely remove)
11344 * clearing/loading VMCS again due to premature trips to ring-3 above.
11345 */
11346 if (pVmxTransient->fIsNestedGuest)
11347 {
11348 if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
11349 {
11350 /*
11351 * Ensure we have synced everything from the guest VMCS and also flag that
11352 * that we need to export the full (nested) guest-CPU context to the
11353 * nested-guest VMCS.
11354 */
11355 HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11356 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_ALL_GUEST);
11357
11358 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
11359 int rc = hmR0VmxSwitchVmcs(&pVCpu->hm.s.vmx.VmcsInfo, &pVCpu->hm.s.vmx.VmcsInfoNstGst);
11360 if (RT_LIKELY(rc == VINF_SUCCESS))
11361 {
11362 pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = true;
11363 ASMSetFlags(fEFlags);
11364 pVmxTransient->pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
11365
11366 /*
11367 * We use a different VM-exit MSR-store area for the nested-guest. Hence,
11368 * flag that we need to update the host MSR values there. Even if we decide
11369 * in the future to share the VM-exit MSR-store area page with the guest,
11370 * if its content differs, we would have to update the host MSRs anyway.
11371 */
11372 pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
11373 Assert(!pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer); /** @todo NSTVMX: Paranoia remove later. */
11374 }
11375 else
11376 {
11377 ASMSetFlags(fEFlags);
11378 return rc;
11379 }
11380 }
11381
11382 /*
11383 * Merge guest VMCS controls with the nested-guest VMCS controls.
11384 *
11385 * Even if we have not executed the guest prior to this (e.g. when resuming
11386 * from a saved state), we should be okay with merging controls as we
11387 * initialize the guest VMCS controls as part of VM setup phase.
11388 */
11389 if (!pVCpu->hm.s.vmx.fMergedNstGstCtls)
11390 {
11391 int rc = hmR0VmxMergeVmcsNested(pVCpu);
11392 AssertRCReturn(rc, rc);
11393 pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
11394 }
11395 }
11396#endif
11397
11398 /*
11399 * Virtualize memory-mapped accesses to the physical APIC (may take locks).
11400 */
11401 PVM pVM = pVCpu->CTX_SUFF(pVM);
11402 if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
11403 && hmR0VmxIsProcCtls2Set(pVCpu, pVmxTransient, VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
11404 && PDMHasApic(pVM))
11405 {
11406 int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
11407 AssertRCReturn(rc, rc);
11408 }
11409
11410 /*
11411 * Evaluate events to be injected into the guest.
11412 *
11413 * Events in TRPM can be injected without inspecting the guest state.
11414 * If any new events (interrupts/NMI) are pending currently, we try to set up the
11415 * guest to cause a VM-exit the next time they are ready to receive the event.
11416 *
11417 * With nested-guests, evaluating pending events may cause VM-exits.
11418 */
11419 if (TRPMHasTrap(pVCpu))
11420 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
11421
11422 uint32_t fIntrState;
11423 rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
11424
11425#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11426 /*
11427 * While evaluating pending events if something failed (unlikely) or if we were
11428 * preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
11429 */
11430 if (rcStrict != VINF_SUCCESS)
11431 return rcStrict;
11432 if ( pVmxTransient->fIsNestedGuest
11433 && !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
11434 return VINF_VMX_VMEXIT;
11435#else
11436 Assert(rcStrict == VINF_SUCCESS);
11437#endif
11438
11439 /*
11440 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
11441 * needs to be done with longjmps or interrupts + preemption enabled. Event injection might
11442 * also result in triple-faulting the VM.
11443 *
11444 * With nested-guests, the above does not apply since unrestricted guest execution is a
11445 * requirement. Regardless, we do this here to avoid duplicating code elsewhere.
11446 */
11447 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
11448 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11449 { /* likely */ }
11450 else
11451 {
11452 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
11453 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11454 return rcStrict;
11455 }
11456
11457 /*
11458 * A longjump might result in importing CR3 even for VM-exits that don't necessarily
11459 * import CR3 themselves. We will need to update them here, as even as late as the above
11460 * hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
11461 * the below force flags to be set.
11462 */
11463 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
11464 {
11465 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
11466 int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
11467 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
11468 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
11469 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
11470 }
11471 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
11472 {
11473 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
11474 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
11475 }
11476
11477#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11478 /* Paranoia. */
11479 Assert(!pVmxTransient->fIsNestedGuest || CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11480#endif
11481
11482 /*
11483 * No longjmps to ring-3 from this point on!!!
11484 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
11485 * This also disables flushing of the R0-logger instance (if any).
11486 */
11487 VMMRZCallRing3Disable(pVCpu);
11488
11489 /*
11490 * Export the guest state bits.
11491 *
11492 * We cannot perform longjmps while loading the guest state because we do not preserve the
11493 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
11494 * CPU migration.
11495 *
11496 * If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
11497 * registers. Hence, exporting of the guest state needs to be done -after- injection of events.
11498 */
11499 rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
11500 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11501 { /* likely */ }
11502 else
11503 {
11504 VMMRZCallRing3Enable(pVCpu);
11505 return rcStrict;
11506 }
11507
11508 /*
11509 * We disable interrupts so that we don't miss any interrupts that would flag preemption
11510 * (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
11511 * preemption disabled for a while. Since this is purely to aid the
11512 * RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
11513 * disable interrupt on NT.
11514 *
11515 * We need to check for force-flags that could've possible been altered since we last
11516 * checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
11517 * see @bugref{6398}).
11518 *
11519 * We also check a couple of other force-flags as a last opportunity to get the EMT back
11520 * to ring-3 before executing guest code.
11521 */
11522 pVmxTransient->fEFlags = ASMIntDisableFlags();
11523
11524 if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
11525 && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
11526 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
11527 && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
11528 {
11529 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
11530 {
11531 pVCpu->hm.s.Event.fPending = false;
11532
11533 /*
11534 * We've injected any pending events. This is really the point of no return (to ring-3).
11535 *
11536 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
11537 * returns from this function, so don't enable them here.
11538 */
11539 return VINF_SUCCESS;
11540 }
11541
11542 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
11543 rcStrict = VINF_EM_RAW_INTERRUPT;
11544 }
11545 else
11546 {
11547 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
11548 rcStrict = VINF_EM_RAW_TO_R3;
11549 }
11550
11551 ASMSetFlags(pVmxTransient->fEFlags);
11552 VMMRZCallRing3Enable(pVCpu);
11553
11554 return rcStrict;
11555}
11556
11557
11558/**
11559 * Final preparations before executing guest code using hardware-assisted VMX.
11560 *
11561 * We can no longer get preempted to a different host CPU and there are no returns
11562 * to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
11563 * failures), this function is not intended to fail sans unrecoverable hardware
11564 * errors.
11565 *
11566 * @param pVCpu The cross context virtual CPU structure.
11567 * @param pVmxTransient The VMX-transient structure.
11568 *
11569 * @remarks Called with preemption disabled.
11570 * @remarks No-long-jump zone!!!
11571 */
11572static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11573{
11574 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
11575 Assert(VMMR0IsLogFlushDisabled(pVCpu));
11576 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
11577 Assert(!pVCpu->hm.s.Event.fPending);
11578
11579 /*
11580 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
11581 */
11582 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
11583 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
11584
11585 PVM pVM = pVCpu->CTX_SUFF(pVM);
11586 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11587
11588 if (!CPUMIsGuestFPUStateActive(pVCpu))
11589 {
11590 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
11591 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
11592 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
11593 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
11594 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
11595 }
11596
11597 /*
11598 * Re-save the host state bits as we may've been preempted (only happens when
11599 * thread-context hooks are used or when the VM start function changes).
11600 * The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
11601 * changed the switcher back to 32-bit, we *must* save the 32-bit host state here,
11602 * see @bugref{8432}.
11603 *
11604 * This may also happen when switching to/from a nested-guest VMCS without leaving
11605 * ring-0.
11606 */
11607 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
11608 {
11609 int rc = hmR0VmxExportHostState(pVCpu);
11610 AssertRC(rc);
11611 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
11612 }
11613 Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
11614
11615 /*
11616 * Export the state shared between host and guest (FPU, debug, lazy MSRs).
11617 */
11618 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
11619 hmR0VmxExportSharedState(pVCpu, pVmxTransient);
11620 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
11621
11622 /*
11623 * Store status of the shared guest/host debug state at the time of VM-entry.
11624 */
11625#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
11626 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
11627 {
11628 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
11629 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
11630 }
11631 else
11632#endif
11633 {
11634 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
11635 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
11636 }
11637
11638 /*
11639 * Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
11640 * more than one conditional check. The post-run side of our code shall determine
11641 * if it needs to sync. the virtual APIC TPR with the TPR-shadow.
11642 */
11643 if (pVmcsInfo->pbVirtApic)
11644 pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
11645
11646 /*
11647 * Update the host MSRs values in the VM-exit MSR-load area.
11648 */
11649 if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
11650 {
11651 if (pVmcsInfo->cExitMsrLoad > 0)
11652 hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
11653 pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
11654 }
11655
11656 /*
11657 * Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
11658 * VMX-preemption timer based on the next virtual sync clock deadline.
11659 */
11660 PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
11661 RTCPUID const idCurrentCpu = pHostCpu->idCpu;
11662 if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
11663 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
11664 {
11665 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
11666 pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
11667 }
11668
11669 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
11670 hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
11671 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
11672 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
11673
11674 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
11675
11676 TMNotifyStartOfExecution(pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
11677 as we're about to start executing the guest . */
11678
11679 /*
11680 * Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
11681 *
11682 * This is done this late as updating the TSC offsetting/preemption timer above
11683 * figures out if we can skip intercepting RDTSCP by calculating the number of
11684 * host CPU ticks till the next virtual sync deadline (for the dynamic case).
11685 */
11686 if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
11687 {
11688 if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
11689 {
11690 hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
11691 /* NB: Because we call hmR0VmxAddAutoLoadStoreMsr with fUpdateHostMsr=true,
11692 it's safe even after hmR0VmxUpdateAutoLoadHostMsrs has already been done. */
11693 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
11694 true /* fSetReadWrite */, true /* fUpdateHostMsr */);
11695 AssertRC(rc);
11696 }
11697 else
11698 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
11699 }
11700
11701#ifdef VBOX_STRICT
11702 Assert(pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs);
11703 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
11704 hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
11705 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo));
11706#endif
11707
11708#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
11709 /** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
11710 * Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs,
11711 * see @bugref{9180#c54}. */
11712 uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
11713 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
11714 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
11715#endif
11716}
11717
11718
11719/**
11720 * First C routine invoked after running guest code using hardware-assisted VMX.
11721 *
11722 * @param pVCpu The cross context virtual CPU structure.
11723 * @param pVmxTransient The VMX-transient structure.
11724 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
11725 *
11726 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
11727 *
11728 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
11729 * unconditionally when it is safe to do so.
11730 */
11731static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
11732{
11733 uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
11734
11735 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
11736 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
11737 pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
11738 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
11739 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
11740 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
11741
11742 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11743 if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
11744 {
11745 uint64_t uGstTsc;
11746 if (!pVmxTransient->fIsNestedGuest)
11747 uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
11748 else
11749 {
11750 uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
11751 uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
11752 }
11753 TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
11754 }
11755
11756 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
11757 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
11758 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
11759
11760#if HC_ARCH_BITS == 64
11761 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
11762#endif
11763#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
11764 /* The 64-on-32 switcher maintains VMCS-launch state on its own
11765 and we need to leave it alone here. */
11766 if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
11767 pVmcsInfo->fVmcsState |= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
11768#else
11769 pVmcsInfo->fVmcsState |= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
11770#endif
11771#ifdef VBOX_STRICT
11772 hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
11773#endif
11774 Assert(!ASMIntAreEnabled());
11775 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
11776 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
11777
11778 /*
11779 * Save the basic VM-exit reason and check if the VM-entry failed.
11780 * See Intel spec. 24.9.1 "Basic VM-exit Information".
11781 */
11782 uint32_t uExitReason;
11783 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
11784 AssertRC(rc);
11785 pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
11786 pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
11787
11788 /*
11789 * Check if VMLAUNCH/VMRESUME succeeded.
11790 * If this failed, we cause a guru meditation and cease further execution.
11791 *
11792 * However, if we are executing a nested-guest we might fail if we use the
11793 * fast path rather than fully emulating VMLAUNCH/VMRESUME instruction in IEM.
11794 */
11795 if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
11796 {
11797 /*
11798 * Update the VM-exit history array here even if the VM-entry failed due to:
11799 * - Invalid guest state.
11800 * - MSR loading.
11801 * - Machine-check event.
11802 *
11803 * In any of the above cases we will still have a "valid" VM-exit reason
11804 * despite @a fVMEntryFailed being false.
11805 *
11806 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
11807 *
11808 * Note! We don't have CS or RIP at this point. Will probably address that later
11809 * by amending the history entry added here.
11810 */
11811 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
11812 UINT64_MAX, uHostTsc);
11813
11814 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
11815 {
11816 VMMRZCallRing3Enable(pVCpu);
11817
11818 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
11819 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
11820
11821#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
11822 rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11823 AssertRC(rc);
11824#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
11825 rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
11826 AssertRC(rc);
11827#else
11828 /*
11829 * Import the guest-interruptibility state always as we need it while evaluating
11830 * injecting events on re-entry.
11831 *
11832 * We don't import CR0 (when unrestricted guest execution is unavailable) despite
11833 * checking for real-mode while exporting the state because all bits that cause
11834 * mode changes wrt CR0 are intercepted.
11835 */
11836 rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
11837 AssertRC(rc);
11838#endif
11839
11840 /*
11841 * Sync the TPR shadow with our APIC state.
11842 */
11843 if ( !pVmxTransient->fIsNestedGuest
11844 && (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
11845 {
11846 Assert(pVmcsInfo->pbVirtApic);
11847 if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
11848 {
11849 rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
11850 AssertRC(rc);
11851 ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11852 }
11853 }
11854
11855 Assert(VMMRZCallRing3IsEnabled(pVCpu));
11856 return;
11857 }
11858 }
11859#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11860 else if (pVmxTransient->fIsNestedGuest)
11861 {
11862# if 0
11863 /*
11864 * Copy the VM-instruction error field to the guest VMCS.
11865 */
11866 /** @todo NSTVMX: Verify we're using the fast path. */
11867 uint32_t u32RoVmInstrError;
11868 rc = VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &u32RoVmInstrError);
11869 AssertRCReturn(rc, rc);
11870 PVMXVVMCS pGstVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
11871 pGstVmcs->u32RoVmInstrError = u32RoVmInstrError;
11872 /** @todo NSTVMX: Advance guest RIP and other fast path related restoration. */
11873# else
11874 AssertMsgFailed(("VMLAUNCH/VMRESUME failed but shouldn't happen when VMLAUNCH/VMRESUME was emulated in IEM!\n"));
11875# endif
11876 }
11877#endif
11878 else
11879 Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
11880
11881 VMMRZCallRing3Enable(pVCpu);
11882}
11883
11884
11885/**
11886 * Runs the guest code using hardware-assisted VMX the normal way.
11887 *
11888 * @returns VBox status code.
11889 * @param pVCpu The cross context virtual CPU structure.
11890 * @param pcLoops Pointer to the number of executed loops.
11891 */
11892static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, uint32_t *pcLoops)
11893{
11894 uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11895 Assert(pcLoops);
11896 Assert(*pcLoops <= cMaxResumeLoops);
11897
11898 VMXTRANSIENT VmxTransient;
11899 RT_ZERO(VmxTransient);
11900 VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11901
11902 /* Paranoia. */
11903 Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
11904 Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11905
11906 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11907 for (;;)
11908 {
11909 Assert(!HMR0SuspendPending());
11910 HMVMX_ASSERT_CPU_SAFE(pVCpu);
11911 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11912
11913 /*
11914 * Preparatory work for running nested-guest code, this may force us to
11915 * return to ring-3.
11916 *
11917 * Warning! This bugger disables interrupts on VINF_SUCCESS!
11918 */
11919 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11920 if (rcStrict != VINF_SUCCESS)
11921 break;
11922
11923 /* Interrupts are disabled at this point! */
11924 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11925 int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11926 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11927 /* Interrupts are re-enabled at this point! */
11928
11929 /*
11930 * Check for errors with running the VM (VMLAUNCH/VMRESUME).
11931 */
11932 if (RT_SUCCESS(rcRun))
11933 { /* very likely */ }
11934 else
11935 {
11936 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11937 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11938 return rcRun;
11939 }
11940
11941 /*
11942 * Profile the VM-exit.
11943 */
11944 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11945 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11946 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11947 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11948 HMVMX_START_EXIT_DISPATCH_PROF();
11949
11950 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11951
11952 /*
11953 * Handle the VM-exit.
11954 */
11955#ifdef HMVMX_USE_FUNCTION_TABLE
11956 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
11957#else
11958 rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
11959#endif
11960 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11961 if (rcStrict == VINF_SUCCESS)
11962 {
11963 if (++(*pcLoops) <= cMaxResumeLoops)
11964 continue;
11965 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11966 rcStrict = VINF_EM_RAW_INTERRUPT;
11967 }
11968 break;
11969 }
11970
11971 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11972 return rcStrict;
11973}
11974
11975#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11976/**
11977 * Runs the nested-guest code using hardware-assisted VMX.
11978 *
11979 * @returns VBox status code.
11980 * @param pVCpu The cross context virtual CPU structure.
11981 * @param pcLoops Pointer to the number of executed loops.
11982 *
11983 * @sa hmR0VmxRunGuestCodeNormal.
11984 */
11985static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPU pVCpu, uint32_t *pcLoops)
11986{
11987 uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11988 Assert(pcLoops);
11989 Assert(*pcLoops <= cMaxResumeLoops);
11990 Assert(CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11991
11992 VMXTRANSIENT VmxTransient;
11993 RT_ZERO(VmxTransient);
11994 VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11995 VmxTransient.fIsNestedGuest = true;
11996
11997 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11998 for (;;)
11999 {
12000 Assert(!HMR0SuspendPending());
12001 HMVMX_ASSERT_CPU_SAFE(pVCpu);
12002 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
12003
12004 /*
12005 * Preparatory work for running guest code, this may force us to
12006 * return to ring-3.
12007 *
12008 * Warning! This bugger disables interrupts on VINF_SUCCESS!
12009 */
12010 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
12011 if (rcStrict != VINF_SUCCESS)
12012 break;
12013
12014 /* Interrupts are disabled at this point! */
12015 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
12016 int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12017 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12018 /* Interrupts are re-enabled at this point! */
12019
12020 /*
12021 * Check for errors with running the VM (VMLAUNCH/VMRESUME).
12022 */
12023 if (RT_SUCCESS(rcRun))
12024 { /* very likely */ }
12025 else
12026 {
12027 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12028 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12029 return rcRun;
12030 }
12031
12032 /*
12033 * Profile the VM-exit.
12034 */
12035 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12036 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12037 STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12038 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12039 HMVMX_START_EXIT_DISPATCH_PROF();
12040
12041 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12042
12043 /*
12044 * Handle the VM-exit.
12045 */
12046 rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
12047 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12048 if ( rcStrict == VINF_SUCCESS
12049 && CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
12050 {
12051 if (++(*pcLoops) <= cMaxResumeLoops)
12052 continue;
12053 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12054 rcStrict = VINF_EM_RAW_INTERRUPT;
12055 }
12056 break;
12057 }
12058
12059 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12060 return rcStrict;
12061}
12062#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12063
12064
12065/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
12066 * probes.
12067 *
12068 * The following few functions and associated structure contains the bloat
12069 * necessary for providing detailed debug events and dtrace probes as well as
12070 * reliable host side single stepping. This works on the principle of
12071 * "subclassing" the normal execution loop and workers. We replace the loop
12072 * method completely and override selected helpers to add necessary adjustments
12073 * to their core operation.
12074 *
12075 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
12076 * any performance for debug and analysis features.
12077 *
12078 * @{
12079 */
12080
12081/**
12082 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
12083 * the debug run loop.
12084 */
12085typedef struct VMXRUNDBGSTATE
12086{
12087 /** The RIP we started executing at. This is for detecting that we stepped. */
12088 uint64_t uRipStart;
12089 /** The CS we started executing with. */
12090 uint16_t uCsStart;
12091
12092 /** Whether we've actually modified the 1st execution control field. */
12093 bool fModifiedProcCtls : 1;
12094 /** Whether we've actually modified the 2nd execution control field. */
12095 bool fModifiedProcCtls2 : 1;
12096 /** Whether we've actually modified the exception bitmap. */
12097 bool fModifiedXcptBitmap : 1;
12098
12099 /** We desire the modified the CR0 mask to be cleared. */
12100 bool fClearCr0Mask : 1;
12101 /** We desire the modified the CR4 mask to be cleared. */
12102 bool fClearCr4Mask : 1;
12103 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
12104 uint32_t fCpe1Extra;
12105 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
12106 uint32_t fCpe1Unwanted;
12107 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
12108 uint32_t fCpe2Extra;
12109 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
12110 uint32_t bmXcptExtra;
12111 /** The sequence number of the Dtrace provider settings the state was
12112 * configured against. */
12113 uint32_t uDtraceSettingsSeqNo;
12114 /** VM-exits to check (one bit per VM-exit). */
12115 uint32_t bmExitsToCheck[3];
12116
12117 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
12118 uint32_t fProcCtlsInitial;
12119 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
12120 uint32_t fProcCtls2Initial;
12121 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
12122 uint32_t bmXcptInitial;
12123} VMXRUNDBGSTATE;
12124AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
12125typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
12126
12127
12128/**
12129 * Initializes the VMXRUNDBGSTATE structure.
12130 *
12131 * @param pVCpu The cross context virtual CPU structure of the
12132 * calling EMT.
12133 * @param pVmxTransient The VMX-transient structure.
12134 * @param pDbgState The debug state to initialize.
12135 */
12136static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
12137{
12138 pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
12139 pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
12140
12141 pDbgState->fModifiedProcCtls = false;
12142 pDbgState->fModifiedProcCtls2 = false;
12143 pDbgState->fModifiedXcptBitmap = false;
12144 pDbgState->fClearCr0Mask = false;
12145 pDbgState->fClearCr4Mask = false;
12146 pDbgState->fCpe1Extra = 0;
12147 pDbgState->fCpe1Unwanted = 0;
12148 pDbgState->fCpe2Extra = 0;
12149 pDbgState->bmXcptExtra = 0;
12150 pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
12151 pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
12152 pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
12153}
12154
12155
12156/**
12157 * Updates the VMSC fields with changes requested by @a pDbgState.
12158 *
12159 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
12160 * immediately before executing guest code, i.e. when interrupts are disabled.
12161 * We don't check status codes here as we cannot easily assert or return in the
12162 * latter case.
12163 *
12164 * @param pVCpu The cross context virtual CPU structure.
12165 * @param pVmxTransient The VMX-transient structure.
12166 * @param pDbgState The debug state.
12167 */
12168static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
12169{
12170 /*
12171 * Ensure desired flags in VMCS control fields are set.
12172 * (Ignoring write failure here, as we're committed and it's just debug extras.)
12173 *
12174 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
12175 * there should be no stale data in pCtx at this point.
12176 */
12177 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12178 if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
12179 || (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
12180 {
12181 pVmcsInfo->u32ProcCtls |= pDbgState->fCpe1Extra;
12182 pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
12183 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
12184 Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
12185 pDbgState->fModifiedProcCtls = true;
12186 }
12187
12188 if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
12189 {
12190 pVmcsInfo->u32ProcCtls2 |= pDbgState->fCpe2Extra;
12191 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
12192 Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
12193 pDbgState->fModifiedProcCtls2 = true;
12194 }
12195
12196 if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
12197 {
12198 pVmcsInfo->u32XcptBitmap |= pDbgState->bmXcptExtra;
12199 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
12200 Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
12201 pDbgState->fModifiedXcptBitmap = true;
12202 }
12203
12204 if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
12205 {
12206 pVmcsInfo->u64Cr0Mask = 0;
12207 VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, 0);
12208 Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
12209 }
12210
12211 if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
12212 {
12213 pVmcsInfo->u64Cr4Mask = 0;
12214 VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, 0);
12215 Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
12216 }
12217
12218 NOREF(pVCpu);
12219}
12220
12221
12222/**
12223 * Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
12224 * re-entry next time around.
12225 *
12226 * @returns Strict VBox status code (i.e. informational status codes too).
12227 * @param pVCpu The cross context virtual CPU structure.
12228 * @param pVmxTransient The VMX-transient structure.
12229 * @param pDbgState The debug state.
12230 * @param rcStrict The return code from executing the guest using single
12231 * stepping.
12232 */
12233static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
12234 VBOXSTRICTRC rcStrict)
12235{
12236 /*
12237 * Restore VM-exit control settings as we may not reenter this function the
12238 * next time around.
12239 */
12240 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12241
12242 /* We reload the initial value, trigger what we can of recalculations the
12243 next time around. From the looks of things, that's all that's required atm. */
12244 if (pDbgState->fModifiedProcCtls)
12245 {
12246 if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
12247 pDbgState->fProcCtlsInitial |= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
12248 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
12249 AssertRCReturn(rc2, rc2);
12250 pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
12251 }
12252
12253 /* We're currently the only ones messing with this one, so just restore the
12254 cached value and reload the field. */
12255 if ( pDbgState->fModifiedProcCtls2
12256 && pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
12257 {
12258 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
12259 AssertRCReturn(rc2, rc2);
12260 pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
12261 }
12262
12263 /* If we've modified the exception bitmap, we restore it and trigger
12264 reloading and partial recalculation the next time around. */
12265 if (pDbgState->fModifiedXcptBitmap)
12266 pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
12267
12268 return rcStrict;
12269}
12270
12271
12272/**
12273 * Configures VM-exit controls for current DBGF and DTrace settings.
12274 *
12275 * This updates @a pDbgState and the VMCS execution control fields to reflect
12276 * the necessary VM-exits demanded by DBGF and DTrace.
12277 *
12278 * @param pVCpu The cross context virtual CPU structure.
12279 * @param pVmxTransient The VMX-transient structure. May update
12280 * fUpdatedTscOffsettingAndPreemptTimer.
12281 * @param pDbgState The debug state.
12282 */
12283static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
12284{
12285 /*
12286 * Take down the dtrace serial number so we can spot changes.
12287 */
12288 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
12289 ASMCompilerBarrier();
12290
12291 /*
12292 * We'll rebuild most of the middle block of data members (holding the
12293 * current settings) as we go along here, so start by clearing it all.
12294 */
12295 pDbgState->bmXcptExtra = 0;
12296 pDbgState->fCpe1Extra = 0;
12297 pDbgState->fCpe1Unwanted = 0;
12298 pDbgState->fCpe2Extra = 0;
12299 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
12300 pDbgState->bmExitsToCheck[i] = 0;
12301
12302 /*
12303 * Software interrupts (INT XXh) - no idea how to trigger these...
12304 */
12305 PVM pVM = pVCpu->CTX_SUFF(pVM);
12306 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
12307 || VBOXVMM_INT_SOFTWARE_ENABLED())
12308 {
12309 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
12310 }
12311
12312 /*
12313 * INT3 breakpoints - triggered by #BP exceptions.
12314 */
12315 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
12316 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
12317
12318 /*
12319 * Exception bitmap and XCPT events+probes.
12320 */
12321 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
12322 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
12323 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
12324
12325 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
12326 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
12327 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
12328 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
12329 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
12330 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
12331 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
12332 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
12333 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
12334 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
12335 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
12336 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
12337 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
12338 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
12339 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
12340 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
12341 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
12342 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
12343
12344 if (pDbgState->bmXcptExtra)
12345 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
12346
12347 /*
12348 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
12349 *
12350 * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
12351 * So, when adding/changing/removing please don't forget to update it.
12352 *
12353 * Some of the macros are picking up local variables to save horizontal space,
12354 * (being able to see it in a table is the lesser evil here).
12355 */
12356#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
12357 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
12358 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
12359#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
12360 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
12361 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
12362 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
12363 } else do { } while (0)
12364#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
12365 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
12366 { \
12367 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
12368 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
12369 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
12370 } else do { } while (0)
12371#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
12372 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
12373 { \
12374 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
12375 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
12376 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
12377 } else do { } while (0)
12378#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
12379 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
12380 { \
12381 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
12382 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
12383 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
12384 } else do { } while (0)
12385
12386 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
12387 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
12388 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
12389 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
12390 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
12391
12392 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
12393 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
12394 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
12395 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
12396 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
12397 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
12398 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
12399 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
12400 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
12401 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
12402 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
12403 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
12404 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
12405 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
12406 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
12407 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
12408 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
12409 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
12410 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
12411 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
12412 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
12413 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
12414 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
12415 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
12416 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
12417 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
12418 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
12419 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
12420 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
12421 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
12422 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
12423 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
12424 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
12425 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
12426 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
12427 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
12428
12429 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
12430 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
12431 {
12432 int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR4
12433 | CPUMCTX_EXTRN_APIC_TPR);
12434 AssertRC(rc);
12435
12436#if 0 /** @todo fix me */
12437 pDbgState->fClearCr0Mask = true;
12438 pDbgState->fClearCr4Mask = true;
12439#endif
12440 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
12441 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_STORE_EXIT | VMX_PROC_CTLS_CR8_STORE_EXIT;
12442 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
12443 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_LOAD_EXIT | VMX_PROC_CTLS_CR8_LOAD_EXIT;
12444 pDbgState->fCpe1Unwanted |= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
12445 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
12446 require clearing here and in the loop if we start using it. */
12447 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
12448 }
12449 else
12450 {
12451 if (pDbgState->fClearCr0Mask)
12452 {
12453 pDbgState->fClearCr0Mask = false;
12454 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12455 }
12456 if (pDbgState->fClearCr4Mask)
12457 {
12458 pDbgState->fClearCr4Mask = false;
12459 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
12460 }
12461 }
12462 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
12463 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
12464
12465 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
12466 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
12467 {
12468 /** @todo later, need to fix handler as it assumes this won't usually happen. */
12469 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
12470 }
12471 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
12472 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
12473
12474 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
12475 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
12476 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
12477 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
12478 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
12479 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
12480 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
12481 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
12482#if 0 /** @todo too slow, fix handler. */
12483 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
12484#endif
12485 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
12486
12487 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
12488 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
12489 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
12490 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
12491 {
12492 pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
12493 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
12494 }
12495 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
12496 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
12497 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
12498 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
12499
12500 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
12501 || IS_EITHER_ENABLED(pVM, INSTR_STR)
12502 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
12503 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
12504 {
12505 pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
12506 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
12507 }
12508 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
12509 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
12510 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
12511 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
12512
12513 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
12514 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
12515 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
12516 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
12517 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
12518 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
12519 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
12520 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
12521 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
12522 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
12523 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
12524 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
12525 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
12526 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
12527 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
12528 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
12529 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
12530 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
12531 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
12532 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
12533 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
12534 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
12535
12536#undef IS_EITHER_ENABLED
12537#undef SET_ONLY_XBM_IF_EITHER_EN
12538#undef SET_CPE1_XBM_IF_EITHER_EN
12539#undef SET_CPEU_XBM_IF_EITHER_EN
12540#undef SET_CPE2_XBM_IF_EITHER_EN
12541
12542 /*
12543 * Sanitize the control stuff.
12544 */
12545 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
12546 if (pDbgState->fCpe2Extra)
12547 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
12548 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
12549 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
12550 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
12551 {
12552 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
12553 pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
12554 }
12555
12556 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
12557 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
12558 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
12559 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
12560}
12561
12562
12563/**
12564 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
12565 * appropriate.
12566 *
12567 * The caller has checked the VM-exit against the
12568 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
12569 * already, so we don't have to do that either.
12570 *
12571 * @returns Strict VBox status code (i.e. informational status codes too).
12572 * @param pVCpu The cross context virtual CPU structure.
12573 * @param pVmxTransient The VMX-transient structure.
12574 * @param uExitReason The VM-exit reason.
12575 *
12576 * @remarks The name of this function is displayed by dtrace, so keep it short
12577 * and to the point. No longer than 33 chars long, please.
12578 */
12579static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
12580{
12581 /*
12582 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
12583 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
12584 *
12585 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
12586 * does. Must add/change/remove both places. Same ordering, please.
12587 *
12588 * Added/removed events must also be reflected in the next section
12589 * where we dispatch dtrace events.
12590 */
12591 bool fDtrace1 = false;
12592 bool fDtrace2 = false;
12593 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
12594 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
12595 uint32_t uEventArg = 0;
12596#define SET_EXIT(a_EventSubName) \
12597 do { \
12598 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
12599 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
12600 } while (0)
12601#define SET_BOTH(a_EventSubName) \
12602 do { \
12603 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
12604 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
12605 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
12606 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
12607 } while (0)
12608 switch (uExitReason)
12609 {
12610 case VMX_EXIT_MTF:
12611 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
12612
12613 case VMX_EXIT_XCPT_OR_NMI:
12614 {
12615 uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12616 switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
12617 {
12618 case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
12619 case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
12620 case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
12621 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
12622 {
12623 if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
12624 {
12625 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12626 uEventArg = pVmxTransient->uExitIntErrorCode;
12627 }
12628 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
12629 switch (enmEvent1)
12630 {
12631 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
12632 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
12633 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
12634 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
12635 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
12636 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
12637 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
12638 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
12639 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
12640 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
12641 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
12642 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
12643 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
12644 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
12645 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
12646 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
12647 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
12648 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
12649 default: break;
12650 }
12651 }
12652 else
12653 AssertFailed();
12654 break;
12655
12656 case VMX_EXIT_INT_INFO_TYPE_SW_INT:
12657 uEventArg = idxVector;
12658 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
12659 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
12660 break;
12661 }
12662 break;
12663 }
12664
12665 case VMX_EXIT_TRIPLE_FAULT:
12666 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
12667 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
12668 break;
12669 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
12670 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
12671 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
12672 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
12673 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
12674
12675 /* Instruction specific VM-exits: */
12676 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
12677 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
12678 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
12679 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
12680 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
12681 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
12682 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
12683 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
12684 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
12685 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
12686 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
12687 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
12688 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
12689 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
12690 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
12691 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
12692 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
12693 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
12694 case VMX_EXIT_MOV_CRX:
12695 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12696 if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
12697 SET_BOTH(CRX_READ);
12698 else
12699 SET_BOTH(CRX_WRITE);
12700 uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
12701 break;
12702 case VMX_EXIT_MOV_DRX:
12703 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12704 if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
12705 == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
12706 SET_BOTH(DRX_READ);
12707 else
12708 SET_BOTH(DRX_WRITE);
12709 uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
12710 break;
12711 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
12712 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
12713 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
12714 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
12715 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
12716 case VMX_EXIT_GDTR_IDTR_ACCESS:
12717 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12718 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
12719 {
12720 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
12721 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
12722 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
12723 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
12724 }
12725 break;
12726
12727 case VMX_EXIT_LDTR_TR_ACCESS:
12728 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12729 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
12730 {
12731 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
12732 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
12733 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
12734 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
12735 }
12736 break;
12737
12738 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
12739 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
12740 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
12741 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
12742 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
12743 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
12744 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
12745 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
12746 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
12747 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
12748 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
12749
12750 /* Events that aren't relevant at this point. */
12751 case VMX_EXIT_EXT_INT:
12752 case VMX_EXIT_INT_WINDOW:
12753 case VMX_EXIT_NMI_WINDOW:
12754 case VMX_EXIT_TPR_BELOW_THRESHOLD:
12755 case VMX_EXIT_PREEMPT_TIMER:
12756 case VMX_EXIT_IO_INSTR:
12757 break;
12758
12759 /* Errors and unexpected events. */
12760 case VMX_EXIT_INIT_SIGNAL:
12761 case VMX_EXIT_SIPI:
12762 case VMX_EXIT_IO_SMI:
12763 case VMX_EXIT_SMI:
12764 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12765 case VMX_EXIT_ERR_MSR_LOAD:
12766 case VMX_EXIT_ERR_MACHINE_CHECK:
12767 case VMX_EXIT_PML_FULL:
12768 case VMX_EXIT_VIRTUALIZED_EOI:
12769 break;
12770
12771 default:
12772 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12773 break;
12774 }
12775#undef SET_BOTH
12776#undef SET_EXIT
12777
12778 /*
12779 * Dtrace tracepoints go first. We do them here at once so we don't
12780 * have to copy the guest state saving and stuff a few dozen times.
12781 * Down side is that we've got to repeat the switch, though this time
12782 * we use enmEvent since the probes are a subset of what DBGF does.
12783 */
12784 if (fDtrace1 || fDtrace2)
12785 {
12786 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12787 hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12788 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12789 switch (enmEvent1)
12790 {
12791 /** @todo consider which extra parameters would be helpful for each probe. */
12792 case DBGFEVENT_END: break;
12793 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
12794 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
12795 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
12796 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
12797 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
12798 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
12799 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
12800 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
12801 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
12802 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
12803 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
12804 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
12805 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
12806 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
12807 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
12808 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
12809 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
12810 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
12811 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12812 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
12813 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
12814 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
12815 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
12816 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
12817 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
12818 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
12819 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
12820 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12821 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12822 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12823 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12824 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
12825 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
12826 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
12827 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
12828 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
12829 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
12830 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
12831 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
12832 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
12833 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
12834 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
12835 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
12836 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
12837 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
12838 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
12839 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
12840 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
12841 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
12842 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
12843 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
12844 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
12845 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
12846 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
12847 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
12848 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
12849 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
12850 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
12851 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
12852 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
12853 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
12854 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
12855 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
12856 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
12857 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
12858 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
12859 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
12860 }
12861 switch (enmEvent2)
12862 {
12863 /** @todo consider which extra parameters would be helpful for each probe. */
12864 case DBGFEVENT_END: break;
12865 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
12866 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
12867 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
12868 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
12869 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
12870 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
12871 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
12872 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
12873 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
12874 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12875 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12876 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12877 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12878 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
12879 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
12880 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
12881 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
12882 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
12883 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
12884 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
12885 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
12886 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
12887 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
12888 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
12889 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
12890 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
12891 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
12892 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
12893 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
12894 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
12895 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
12896 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
12897 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
12898 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
12899 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
12900 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
12901 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
12902 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
12903 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
12904 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
12905 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
12906 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
12907 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
12908 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
12909 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
12910 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
12911 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
12912 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
12913 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
12914 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
12915 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
12916 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
12917 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
12918 }
12919 }
12920
12921 /*
12922 * Fire of the DBGF event, if enabled (our check here is just a quick one,
12923 * the DBGF call will do a full check).
12924 *
12925 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
12926 * Note! If we have to events, we prioritize the first, i.e. the instruction
12927 * one, in order to avoid event nesting.
12928 */
12929 PVM pVM = pVCpu->CTX_SUFF(pVM);
12930 if ( enmEvent1 != DBGFEVENT_END
12931 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
12932 {
12933 hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
12934 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
12935 if (rcStrict != VINF_SUCCESS)
12936 return rcStrict;
12937 }
12938 else if ( enmEvent2 != DBGFEVENT_END
12939 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
12940 {
12941 hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
12942 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
12943 if (rcStrict != VINF_SUCCESS)
12944 return rcStrict;
12945 }
12946
12947 return VINF_SUCCESS;
12948}
12949
12950
12951/**
12952 * Single-stepping VM-exit filtering.
12953 *
12954 * This is preprocessing the VM-exits and deciding whether we've gotten far
12955 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
12956 * handling is performed.
12957 *
12958 * @returns Strict VBox status code (i.e. informational status codes too).
12959 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
12960 * @param pVmxTransient The VMX-transient structure.
12961 * @param pDbgState The debug state.
12962 */
12963DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
12964{
12965 /*
12966 * Expensive (saves context) generic dtrace VM-exit probe.
12967 */
12968 uint32_t const uExitReason = pVmxTransient->uExitReason;
12969 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
12970 { /* more likely */ }
12971 else
12972 {
12973 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12974 int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12975 AssertRC(rc);
12976 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
12977 }
12978
12979 /*
12980 * Check for host NMI, just to get that out of the way.
12981 */
12982 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
12983 { /* normally likely */ }
12984 else
12985 {
12986 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12987 AssertRCReturn(rc2, rc2);
12988 uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12989 if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12990 return hmR0VmxExitHostNmi(pVCpu);
12991 }
12992
12993 /*
12994 * Check for single stepping event if we're stepping.
12995 */
12996 if (pVCpu->hm.s.fSingleInstruction)
12997 {
12998 switch (uExitReason)
12999 {
13000 case VMX_EXIT_MTF:
13001 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
13002
13003 /* Various events: */
13004 case VMX_EXIT_XCPT_OR_NMI:
13005 case VMX_EXIT_EXT_INT:
13006 case VMX_EXIT_TRIPLE_FAULT:
13007 case VMX_EXIT_INT_WINDOW:
13008 case VMX_EXIT_NMI_WINDOW:
13009 case VMX_EXIT_TASK_SWITCH:
13010 case VMX_EXIT_TPR_BELOW_THRESHOLD:
13011 case VMX_EXIT_APIC_ACCESS:
13012 case VMX_EXIT_EPT_VIOLATION:
13013 case VMX_EXIT_EPT_MISCONFIG:
13014 case VMX_EXIT_PREEMPT_TIMER:
13015
13016 /* Instruction specific VM-exits: */
13017 case VMX_EXIT_CPUID:
13018 case VMX_EXIT_GETSEC:
13019 case VMX_EXIT_HLT:
13020 case VMX_EXIT_INVD:
13021 case VMX_EXIT_INVLPG:
13022 case VMX_EXIT_RDPMC:
13023 case VMX_EXIT_RDTSC:
13024 case VMX_EXIT_RSM:
13025 case VMX_EXIT_VMCALL:
13026 case VMX_EXIT_VMCLEAR:
13027 case VMX_EXIT_VMLAUNCH:
13028 case VMX_EXIT_VMPTRLD:
13029 case VMX_EXIT_VMPTRST:
13030 case VMX_EXIT_VMREAD:
13031 case VMX_EXIT_VMRESUME:
13032 case VMX_EXIT_VMWRITE:
13033 case VMX_EXIT_VMXOFF:
13034 case VMX_EXIT_VMXON:
13035 case VMX_EXIT_MOV_CRX:
13036 case VMX_EXIT_MOV_DRX:
13037 case VMX_EXIT_IO_INSTR:
13038 case VMX_EXIT_RDMSR:
13039 case VMX_EXIT_WRMSR:
13040 case VMX_EXIT_MWAIT:
13041 case VMX_EXIT_MONITOR:
13042 case VMX_EXIT_PAUSE:
13043 case VMX_EXIT_GDTR_IDTR_ACCESS:
13044 case VMX_EXIT_LDTR_TR_ACCESS:
13045 case VMX_EXIT_INVEPT:
13046 case VMX_EXIT_RDTSCP:
13047 case VMX_EXIT_INVVPID:
13048 case VMX_EXIT_WBINVD:
13049 case VMX_EXIT_XSETBV:
13050 case VMX_EXIT_RDRAND:
13051 case VMX_EXIT_INVPCID:
13052 case VMX_EXIT_VMFUNC:
13053 case VMX_EXIT_RDSEED:
13054 case VMX_EXIT_XSAVES:
13055 case VMX_EXIT_XRSTORS:
13056 {
13057 int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
13058 AssertRCReturn(rc, rc);
13059 if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
13060 || pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
13061 return VINF_EM_DBG_STEPPED;
13062 break;
13063 }
13064
13065 /* Errors and unexpected events: */
13066 case VMX_EXIT_INIT_SIGNAL:
13067 case VMX_EXIT_SIPI:
13068 case VMX_EXIT_IO_SMI:
13069 case VMX_EXIT_SMI:
13070 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
13071 case VMX_EXIT_ERR_MSR_LOAD:
13072 case VMX_EXIT_ERR_MACHINE_CHECK:
13073 case VMX_EXIT_PML_FULL:
13074 case VMX_EXIT_VIRTUALIZED_EOI:
13075 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
13076 break;
13077
13078 default:
13079 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
13080 break;
13081 }
13082 }
13083
13084 /*
13085 * Check for debugger event breakpoints and dtrace probes.
13086 */
13087 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
13088 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
13089 {
13090 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
13091 if (rcStrict != VINF_SUCCESS)
13092 return rcStrict;
13093 }
13094
13095 /*
13096 * Normal processing.
13097 */
13098#ifdef HMVMX_USE_FUNCTION_TABLE
13099 return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
13100#else
13101 return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
13102#endif
13103}
13104
13105
13106/**
13107 * Single steps guest code using hardware-assisted VMX.
13108 *
13109 * This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
13110 * but single-stepping through the hypervisor debugger.
13111 *
13112 * @returns Strict VBox status code (i.e. informational status codes too).
13113 * @param pVCpu The cross context virtual CPU structure.
13114 * @param pcLoops Pointer to the number of executed loops.
13115 *
13116 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
13117 */
13118static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, uint32_t *pcLoops)
13119{
13120 uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
13121 Assert(pcLoops);
13122 Assert(*pcLoops <= cMaxResumeLoops);
13123
13124 VMXTRANSIENT VmxTransient;
13125 RT_ZERO(VmxTransient);
13126 VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
13127
13128 /* Set HMCPU indicators. */
13129 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
13130 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
13131 pVCpu->hm.s.fDebugWantRdTscExit = false;
13132 pVCpu->hm.s.fUsingDebugLoop = true;
13133
13134 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
13135 VMXRUNDBGSTATE DbgState;
13136 hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
13137 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
13138
13139 /*
13140 * The loop.
13141 */
13142 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
13143 for (;;)
13144 {
13145 Assert(!HMR0SuspendPending());
13146 HMVMX_ASSERT_CPU_SAFE(pVCpu);
13147 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
13148 bool fStepping = pVCpu->hm.s.fSingleInstruction;
13149
13150 /* Set up VM-execution controls the next two can respond to. */
13151 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
13152
13153 /*
13154 * Preparatory work for running guest code, this may force us to
13155 * return to ring-3.
13156 *
13157 * Warning! This bugger disables interrupts on VINF_SUCCESS!
13158 */
13159 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
13160 if (rcStrict != VINF_SUCCESS)
13161 break;
13162
13163 /* Interrupts are disabled at this point! */
13164 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
13165
13166 /* Override any obnoxious code in the above two calls. */
13167 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
13168
13169 /*
13170 * Finally execute the guest.
13171 */
13172 int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
13173
13174 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
13175 /* Interrupts are re-enabled at this point! */
13176
13177 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
13178 if (RT_SUCCESS(rcRun))
13179 { /* very likely */ }
13180 else
13181 {
13182 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
13183 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
13184 return rcRun;
13185 }
13186
13187 /* Profile the VM-exit. */
13188 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
13189 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
13190 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
13191 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
13192 HMVMX_START_EXIT_DISPATCH_PROF();
13193
13194 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
13195
13196 /*
13197 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
13198 */
13199 rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
13200 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
13201 if (rcStrict != VINF_SUCCESS)
13202 break;
13203 if (++(*pcLoops) > cMaxResumeLoops)
13204 {
13205 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
13206 rcStrict = VINF_EM_RAW_INTERRUPT;
13207 break;
13208 }
13209
13210 /*
13211 * Stepping: Did the RIP change, if so, consider it a single step.
13212 * Otherwise, make sure one of the TFs gets set.
13213 */
13214 if (fStepping)
13215 {
13216 int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
13217 AssertRC(rc);
13218 if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
13219 || pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
13220 {
13221 rcStrict = VINF_EM_DBG_STEPPED;
13222 break;
13223 }
13224 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
13225 }
13226
13227 /*
13228 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
13229 */
13230 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
13231 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
13232 }
13233
13234 /*
13235 * Clear the X86_EFL_TF if necessary.
13236 */
13237 if (pVCpu->hm.s.fClearTrapFlag)
13238 {
13239 int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
13240 AssertRC(rc);
13241 pVCpu->hm.s.fClearTrapFlag = false;
13242 pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
13243 }
13244 /** @todo there seems to be issues with the resume flag when the monitor trap
13245 * flag is pending without being used. Seen early in bios init when
13246 * accessing APIC page in protected mode. */
13247
13248 /*
13249 * Restore VM-exit control settings as we may not re-enter this function the
13250 * next time around.
13251 */
13252 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
13253
13254 /* Restore HMCPU indicators. */
13255 pVCpu->hm.s.fUsingDebugLoop = false;
13256 pVCpu->hm.s.fDebugWantRdTscExit = false;
13257 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
13258
13259 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
13260 return rcStrict;
13261}
13262
13263
13264/** @} */
13265
13266
13267/**
13268 * Checks if any expensive dtrace probes are enabled and we should go to the
13269 * debug loop.
13270 *
13271 * @returns true if we should use debug loop, false if not.
13272 */
13273static bool hmR0VmxAnyExpensiveProbesEnabled(void)
13274{
13275 /* It's probably faster to OR the raw 32-bit counter variables together.
13276 Since the variables are in an array and the probes are next to one
13277 another (more or less), we have good locality. So, better read
13278 eight-nine cache lines ever time and only have one conditional, than
13279 128+ conditionals, right? */
13280 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
13281 | VBOXVMM_XCPT_DE_ENABLED_RAW()
13282 | VBOXVMM_XCPT_DB_ENABLED_RAW()
13283 | VBOXVMM_XCPT_BP_ENABLED_RAW()
13284 | VBOXVMM_XCPT_OF_ENABLED_RAW()
13285 | VBOXVMM_XCPT_BR_ENABLED_RAW()
13286 | VBOXVMM_XCPT_UD_ENABLED_RAW()
13287 | VBOXVMM_XCPT_NM_ENABLED_RAW()
13288 | VBOXVMM_XCPT_DF_ENABLED_RAW()
13289 | VBOXVMM_XCPT_TS_ENABLED_RAW()
13290 | VBOXVMM_XCPT_NP_ENABLED_RAW()
13291 | VBOXVMM_XCPT_SS_ENABLED_RAW()
13292 | VBOXVMM_XCPT_GP_ENABLED_RAW()
13293 | VBOXVMM_XCPT_PF_ENABLED_RAW()
13294 | VBOXVMM_XCPT_MF_ENABLED_RAW()
13295 | VBOXVMM_XCPT_AC_ENABLED_RAW()
13296 | VBOXVMM_XCPT_XF_ENABLED_RAW()
13297 | VBOXVMM_XCPT_VE_ENABLED_RAW()
13298 | VBOXVMM_XCPT_SX_ENABLED_RAW()
13299 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
13300 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
13301 ) != 0
13302 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
13303 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
13304 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
13305 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
13306 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
13307 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
13308 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
13309 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
13310 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
13311 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
13312 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
13313 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
13314 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
13315 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
13316 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
13317 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
13318 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
13319 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
13320 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
13321 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
13322 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
13323 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
13324 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
13325 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
13326 | VBOXVMM_INSTR_STR_ENABLED_RAW()
13327 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
13328 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
13329 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
13330 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
13331 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
13332 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
13333 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
13334 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
13335 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
13336 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
13337 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
13338 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
13339 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
13340 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
13341 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
13342 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
13343 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
13344 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
13345 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
13346 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
13347 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
13348 ) != 0
13349 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
13350 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
13351 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
13352 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
13353 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
13354 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
13355 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
13356 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
13357 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
13358 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
13359 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
13360 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
13361 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
13362 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
13363 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
13364 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
13365 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
13366 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
13367 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
13368 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
13369 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
13370 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
13371 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
13372 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
13373 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
13374 | VBOXVMM_EXIT_STR_ENABLED_RAW()
13375 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
13376 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
13377 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
13378 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
13379 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
13380 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
13381 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
13382 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
13383 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
13384 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
13385 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
13386 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
13387 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
13388 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
13389 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
13390 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
13391 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
13392 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
13393 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
13394 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
13395 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
13396 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
13397 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
13398 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
13399 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
13400 ) != 0;
13401}
13402
13403
13404/**
13405 * Runs the guest using hardware-assisted VMX.
13406 *
13407 * @returns Strict VBox status code (i.e. informational status codes too).
13408 * @param pVCpu The cross context virtual CPU structure.
13409 */
13410VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
13411{
13412 AssertPtr(pVCpu);
13413 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13414 Assert(VMMRZCallRing3IsEnabled(pVCpu));
13415 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
13416 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
13417
13418 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
13419
13420 VBOXSTRICTRC rcStrict;
13421 uint32_t cLoops = 0;
13422#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13423 bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
13424#else
13425 bool const fInNestedGuestMode = false;
13426#endif
13427 if (!fInNestedGuestMode)
13428 {
13429 if ( !pVCpu->hm.s.fUseDebugLoop
13430 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
13431 && !DBGFIsStepping(pVCpu)
13432 && !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
13433 rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
13434 else
13435 rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
13436 }
13437#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13438 else
13439 rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
13440
13441 if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
13442 rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
13443#endif
13444
13445 int const rcLoop = VBOXSTRICTRC_VAL(rcStrict);
13446 switch (rcLoop)
13447 {
13448 case VINF_VMX_VMEXIT: rcStrict = VINF_SUCCESS; break;
13449 case VERR_EM_INTERPRETER: rcStrict = VINF_EM_RAW_EMULATE_INSTR; break;
13450 case VINF_EM_RESET: rcStrict = VINF_EM_TRIPLE_FAULT; break;
13451 }
13452
13453 int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
13454 if (RT_FAILURE(rc2))
13455 {
13456 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
13457 rcStrict = rc2;
13458 }
13459 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
13460 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
13461 return rcStrict;
13462}
13463
13464
13465#ifndef HMVMX_USE_FUNCTION_TABLE
13466/**
13467 * Handles a guest VM-exit from hardware-assisted VMX execution.
13468 *
13469 * @returns Strict VBox status code (i.e. informational status codes too).
13470 * @param pVCpu The cross context virtual CPU structure.
13471 * @param pVmxTransient The VMX-transient structure.
13472 */
13473DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13474{
13475#ifdef DEBUG_ramshankar
13476#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
13477 do { \
13478 if (a_fSave != 0) \
13479 hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL); \
13480 VBOXSTRICTRC rcStrict = a_CallExpr; \
13481 if (a_fSave != 0) \
13482 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
13483 return rcStrict; \
13484 } while (0)
13485#else
13486# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
13487#endif
13488 uint32_t const uExitReason = pVmxTransient->uExitReason;
13489 switch (uExitReason)
13490 {
13491 case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
13492 case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
13493 case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
13494 case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
13495 case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
13496 case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
13497 case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
13498 case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
13499 case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
13500 case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
13501 case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
13502 case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
13503 case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
13504 case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
13505 case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
13506 case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
13507 case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
13508 case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
13509 case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
13510 case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
13511 case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
13512 case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
13513 case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
13514 case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
13515 case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
13516 case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
13517 case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
13518 case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
13519 case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
13520 case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
13521#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13522 case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
13523 case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
13524 case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
13525 case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
13526 case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
13527 case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
13528 case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
13529 case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
13530 case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
13531 case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, hmR0VmxExitInvvpid(pVCpu, pVmxTransient));
13532 case VMX_EXIT_INVEPT: VMEXIT_CALL_RET(0, hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient));
13533#else
13534 case VMX_EXIT_VMCLEAR:
13535 case VMX_EXIT_VMLAUNCH:
13536 case VMX_EXIT_VMPTRLD:
13537 case VMX_EXIT_VMPTRST:
13538 case VMX_EXIT_VMREAD:
13539 case VMX_EXIT_VMRESUME:
13540 case VMX_EXIT_VMWRITE:
13541 case VMX_EXIT_VMXOFF:
13542 case VMX_EXIT_VMXON:
13543 case VMX_EXIT_INVVPID:
13544 case VMX_EXIT_INVEPT:
13545 return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
13546#endif
13547
13548 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
13549 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
13550 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
13551
13552 case VMX_EXIT_INIT_SIGNAL:
13553 case VMX_EXIT_SIPI:
13554 case VMX_EXIT_IO_SMI:
13555 case VMX_EXIT_SMI:
13556 case VMX_EXIT_ERR_MSR_LOAD:
13557 case VMX_EXIT_ERR_MACHINE_CHECK:
13558 case VMX_EXIT_PML_FULL:
13559 case VMX_EXIT_VIRTUALIZED_EOI:
13560 case VMX_EXIT_GDTR_IDTR_ACCESS:
13561 case VMX_EXIT_LDTR_TR_ACCESS:
13562 case VMX_EXIT_APIC_WRITE:
13563 case VMX_EXIT_RDRAND:
13564 case VMX_EXIT_RSM:
13565 case VMX_EXIT_VMFUNC:
13566 case VMX_EXIT_ENCLS:
13567 case VMX_EXIT_RDSEED:
13568 case VMX_EXIT_XSAVES:
13569 case VMX_EXIT_XRSTORS:
13570 case VMX_EXIT_UMWAIT:
13571 case VMX_EXIT_TPAUSE:
13572 default:
13573 return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
13574 }
13575#undef VMEXIT_CALL_RET
13576}
13577#endif /* !HMVMX_USE_FUNCTION_TABLE */
13578
13579
13580#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13581/**
13582 * Handles a nested-guest VM-exit from hardware-assisted VMX execution.
13583 *
13584 * @returns Strict VBox status code (i.e. informational status codes too).
13585 * @param pVCpu The cross context virtual CPU structure.
13586 * @param pVmxTransient The VMX-transient structure.
13587 */
13588DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13589{
13590 uint32_t const uExitReason = pVmxTransient->uExitReason;
13591 switch (uExitReason)
13592 {
13593 case VMX_EXIT_EPT_MISCONFIG: return hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient);
13594 case VMX_EXIT_EPT_VIOLATION: return hmR0VmxExitEptViolation(pVCpu, pVmxTransient);
13595 case VMX_EXIT_XCPT_OR_NMI: return hmR0VmxExitXcptOrNmiNested(pVCpu, pVmxTransient);
13596 case VMX_EXIT_IO_INSTR: return hmR0VmxExitIoInstrNested(pVCpu, pVmxTransient);
13597 case VMX_EXIT_HLT: return hmR0VmxExitHltNested(pVCpu, pVmxTransient);
13598
13599 /*
13600 * We shouldn't direct host physical interrupts to the nested-guest.
13601 */
13602 case VMX_EXIT_EXT_INT:
13603 return hmR0VmxExitExtInt(pVCpu, pVmxTransient);
13604
13605 /*
13606 * Instructions that cause VM-exits unconditionally or the condition is
13607 * always is taken solely from the guest hypervisor (meaning if the VM-exit
13608 * happens, it's guaranteed to be a nested-guest VM-exit).
13609 *
13610 * - Provides VM-exit instruction length ONLY.
13611 */
13612 case VMX_EXIT_CPUID: /* Unconditional. */
13613 case VMX_EXIT_VMCALL:
13614 case VMX_EXIT_GETSEC:
13615 case VMX_EXIT_INVD:
13616 case VMX_EXIT_XSETBV:
13617 case VMX_EXIT_VMLAUNCH:
13618 case VMX_EXIT_VMRESUME:
13619 case VMX_EXIT_VMXOFF:
13620 case VMX_EXIT_ENCLS: /* Condition specified solely by guest hypervisor. */
13621 case VMX_EXIT_VMFUNC:
13622 return hmR0VmxExitInstrNested(pVCpu, pVmxTransient);
13623
13624 /*
13625 * Instructions that cause VM-exits unconditionally or the condition is
13626 * always is taken solely from the guest hypervisor (meaning if the VM-exit
13627 * happens, it's guaranteed to be a nested-guest VM-exit).
13628 *
13629 * - Provides VM-exit instruction length.
13630 * - Provides VM-exit information.
13631 * - Optionally provides Exit qualification.
13632 *
13633 * Since Exit qualification is 0 for all VM-exits where it is not
13634 * applicable, reading and passing it to the guest should produce
13635 * defined behavior.
13636 *
13637 * See Intel spec. 27.2.1 "Basic VM-Exit Information".
13638 */
13639 case VMX_EXIT_INVEPT: /* Unconditional. */
13640 case VMX_EXIT_INVVPID:
13641 case VMX_EXIT_VMCLEAR:
13642 case VMX_EXIT_VMPTRLD:
13643 case VMX_EXIT_VMPTRST:
13644 case VMX_EXIT_VMXON:
13645 case VMX_EXIT_GDTR_IDTR_ACCESS: /* Condition specified solely by guest hypervisor. */
13646 case VMX_EXIT_LDTR_TR_ACCESS:
13647 case VMX_EXIT_RDRAND:
13648 case VMX_EXIT_RDSEED:
13649 case VMX_EXIT_XSAVES:
13650 case VMX_EXIT_XRSTORS:
13651 case VMX_EXIT_UMWAIT:
13652 case VMX_EXIT_TPAUSE:
13653 return hmR0VmxExitInstrWithInfoNested(pVCpu, pVmxTransient);
13654
13655 case VMX_EXIT_RDTSC: return hmR0VmxExitRdtscNested(pVCpu, pVmxTransient);
13656 case VMX_EXIT_RDTSCP: return hmR0VmxExitRdtscpNested(pVCpu, pVmxTransient);
13657 case VMX_EXIT_RDMSR: return hmR0VmxExitRdmsrNested(pVCpu, pVmxTransient);
13658 case VMX_EXIT_WRMSR: return hmR0VmxExitWrmsrNested(pVCpu, pVmxTransient);
13659 case VMX_EXIT_INVLPG: return hmR0VmxExitInvlpgNested(pVCpu, pVmxTransient);
13660 case VMX_EXIT_INVPCID: return hmR0VmxExitInvpcidNested(pVCpu, pVmxTransient);
13661 case VMX_EXIT_TASK_SWITCH: return hmR0VmxExitTaskSwitchNested(pVCpu, pVmxTransient);
13662 case VMX_EXIT_WBINVD: return hmR0VmxExitWbinvdNested(pVCpu, pVmxTransient);
13663 case VMX_EXIT_MTF: return hmR0VmxExitMtfNested(pVCpu, pVmxTransient);
13664 case VMX_EXIT_APIC_ACCESS: return hmR0VmxExitApicAccessNested(pVCpu, pVmxTransient);
13665 case VMX_EXIT_APIC_WRITE: return hmR0VmxExitApicWriteNested(pVCpu, pVmxTransient);
13666 case VMX_EXIT_VIRTUALIZED_EOI: return hmR0VmxExitVirtEoiNested(pVCpu, pVmxTransient);
13667 case VMX_EXIT_MOV_CRX: return hmR0VmxExitMovCRxNested(pVCpu, pVmxTransient);
13668 case VMX_EXIT_INT_WINDOW: return hmR0VmxExitIntWindowNested(pVCpu, pVmxTransient);
13669 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindowNested(pVCpu, pVmxTransient);
13670 case VMX_EXIT_TPR_BELOW_THRESHOLD: return hmR0VmxExitTprBelowThresholdNested(pVCpu, pVmxTransient);
13671 case VMX_EXIT_MWAIT: return hmR0VmxExitMwaitNested(pVCpu, pVmxTransient);
13672 case VMX_EXIT_MONITOR: return hmR0VmxExitMonitorNested(pVCpu, pVmxTransient);
13673 case VMX_EXIT_PAUSE: return hmR0VmxExitPauseNested(pVCpu, pVmxTransient);
13674
13675 case VMX_EXIT_PREEMPT_TIMER:
13676 {
13677 /** @todo NSTVMX: Preempt timer. */
13678 return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
13679 }
13680
13681 case VMX_EXIT_MOV_DRX: return hmR0VmxExitMovDRxNested(pVCpu, pVmxTransient);
13682 case VMX_EXIT_RDPMC: return hmR0VmxExitRdpmcNested(pVCpu, pVmxTransient);
13683
13684 case VMX_EXIT_VMREAD:
13685 case VMX_EXIT_VMWRITE: return hmR0VmxExitVmreadVmwriteNested(pVCpu, pVmxTransient);
13686
13687 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFaultNested(pVCpu, pVmxTransient);
13688 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestStateNested(pVCpu, pVmxTransient);
13689
13690 case VMX_EXIT_INIT_SIGNAL:
13691 case VMX_EXIT_SIPI:
13692 case VMX_EXIT_IO_SMI:
13693 case VMX_EXIT_SMI:
13694 case VMX_EXIT_ERR_MSR_LOAD:
13695 case VMX_EXIT_ERR_MACHINE_CHECK:
13696 case VMX_EXIT_PML_FULL:
13697 case VMX_EXIT_RSM:
13698 default:
13699 return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
13700 }
13701}
13702#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13703
13704
13705#ifdef VBOX_STRICT
13706/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
13707# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
13708 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
13709
13710# define HMVMX_ASSERT_PREEMPT_CPUID() \
13711 do { \
13712 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
13713 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
13714 } while (0)
13715
13716# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13717 do { \
13718 AssertPtr((a_pVCpu)); \
13719 AssertPtr((a_pVmxTransient)); \
13720 Assert((a_pVmxTransient)->fVMEntryFailed == false); \
13721 Assert((a_pVmxTransient)->pVmcsInfo); \
13722 Assert(ASMIntAreEnabled()); \
13723 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
13724 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
13725 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
13726 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
13727 if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
13728 HMVMX_ASSERT_PREEMPT_CPUID(); \
13729 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
13730 } while (0)
13731
13732# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13733 do { \
13734 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); \
13735 Assert((a_pVmxTransient)->fIsNestedGuest); \
13736 } while (0)
13737
13738# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13739 do { \
13740 Log4Func(("\n")); \
13741 } while (0)
13742#else
13743# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13744 do { \
13745 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
13746 NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
13747 } while (0)
13748
13749# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13750 do { HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); } while (0)
13751
13752# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
13753#endif
13754
13755
13756/**
13757 * Advances the guest RIP by the specified number of bytes.
13758 *
13759 * @param pVCpu The cross context virtual CPU structure.
13760 * @param cbInstr Number of bytes to advance the RIP by.
13761 *
13762 * @remarks No-long-jump zone!!!
13763 */
13764DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
13765{
13766 /* Advance the RIP. */
13767 pVCpu->cpum.GstCtx.rip += cbInstr;
13768 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
13769
13770 /* Update interrupt inhibition. */
13771 if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
13772 && pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
13773 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
13774}
13775
13776
13777/**
13778 * Advances the guest RIP after reading it from the VMCS.
13779 *
13780 * @returns VBox status code, no informational status codes.
13781 * @param pVCpu The cross context virtual CPU structure.
13782 * @param pVmxTransient The VMX-transient structure.
13783 *
13784 * @remarks No-long-jump zone!!!
13785 */
13786static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13787{
13788 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13789 rc |= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS);
13790 AssertRCReturn(rc, rc);
13791
13792 hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
13793 return VINF_SUCCESS;
13794}
13795
13796
13797/**
13798 * Handle a condition that occurred while delivering an event through the guest
13799 * IDT.
13800 *
13801 * @returns Strict VBox status code (i.e. informational status codes too).
13802 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
13803 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
13804 * to continue execution of the guest which will delivery the \#DF.
13805 * @retval VINF_EM_RESET if we detected a triple-fault condition.
13806 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
13807 *
13808 * @param pVCpu The cross context virtual CPU structure.
13809 * @param pVmxTransient The VMX-transient structure.
13810 *
13811 * @remarks No-long-jump zone!!!
13812 */
13813static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13814{
13815 /* Read the IDT vectoring info. and VM-exit interruption info. */
13816 {
13817 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
13818 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13819 AssertRCReturn(rc, rc);
13820 }
13821
13822 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
13823 uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13824 if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
13825 {
13826 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
13827 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
13828
13829 /*
13830 * If the event was a software interrupt (generated with INT n) or a software exception
13831 * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
13832 * can handle the VM-exit and continue guest execution which will re-execute the
13833 * instruction rather than re-injecting the exception, as that can cause premature
13834 * trips to ring-3 before injection and involve TRPM which currently has no way of
13835 * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
13836 * the problem).
13837 */
13838 IEMXCPTRAISE enmRaise;
13839 IEMXCPTRAISEINFO fRaiseInfo;
13840 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
13841 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
13842 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
13843 {
13844 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
13845 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13846 }
13847 else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
13848 {
13849 uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
13850 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
13851 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
13852
13853 /** @todo Make AssertMsgReturn as just AssertMsg later. */
13854 AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
13855 ("Unexpected VM-exit interruption vector type %#x!\n", uExitVectorType), VERR_VMX_IPE_5);
13856
13857 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
13858
13859 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
13860 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
13861 {
13862 pVmxTransient->fVectoringPF = true;
13863 enmRaise = IEMXCPTRAISE_PREV_EVENT;
13864 }
13865 }
13866 else
13867 {
13868 /*
13869 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
13870 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
13871 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
13872 */
13873 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
13874 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13875 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
13876 enmRaise = IEMXCPTRAISE_PREV_EVENT;
13877 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13878 }
13879
13880 /*
13881 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
13882 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
13883 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
13884 * subsequent VM-entry would fail.
13885 *
13886 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
13887 */
13888 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13889 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
13890 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
13891 && hmR0VmxIsPinCtlsSet(pVCpu, pVmxTransient, VMX_PIN_CTLS_VIRT_NMI)
13892 && CPUMIsGuestNmiBlocking(pVCpu))
13893 {
13894 CPUMSetGuestNmiBlocking(pVCpu, false);
13895 }
13896
13897 switch (enmRaise)
13898 {
13899 case IEMXCPTRAISE_CURRENT_XCPT:
13900 {
13901 Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
13902 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
13903 Assert(rcStrict == VINF_SUCCESS);
13904 break;
13905 }
13906
13907 case IEMXCPTRAISE_PREV_EVENT:
13908 {
13909 uint32_t u32ErrCode;
13910 if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
13911 {
13912 int rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
13913 AssertRCReturn(rc, rc);
13914 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
13915 }
13916 else
13917 u32ErrCode = 0;
13918
13919 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
13920 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
13921 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
13922 0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
13923
13924 Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
13925 pVCpu->hm.s.Event.u32ErrCode));
13926 Assert(rcStrict == VINF_SUCCESS);
13927 break;
13928 }
13929
13930 case IEMXCPTRAISE_REEXEC_INSTR:
13931 Assert(rcStrict == VINF_SUCCESS);
13932 break;
13933
13934 case IEMXCPTRAISE_DOUBLE_FAULT:
13935 {
13936 /*
13937 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
13938 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
13939 */
13940 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
13941 {
13942 pVmxTransient->fVectoringDoublePF = true;
13943 Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
13944 pVCpu->cpum.GstCtx.cr2));
13945 rcStrict = VINF_SUCCESS;
13946 }
13947 else
13948 {
13949 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
13950 hmR0VmxSetPendingXcptDF(pVCpu);
13951 Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
13952 uIdtVector, uExitVector));
13953 rcStrict = VINF_HM_DOUBLE_FAULT;
13954 }
13955 break;
13956 }
13957
13958 case IEMXCPTRAISE_TRIPLE_FAULT:
13959 {
13960 Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
13961 rcStrict = VINF_EM_RESET;
13962 break;
13963 }
13964
13965 case IEMXCPTRAISE_CPU_HANG:
13966 {
13967 Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
13968 rcStrict = VERR_EM_GUEST_CPU_HANG;
13969 break;
13970 }
13971
13972 default:
13973 {
13974 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
13975 rcStrict = VERR_VMX_IPE_2;
13976 break;
13977 }
13978 }
13979 }
13980 else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
13981 && VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
13982 && uExitVector != X86_XCPT_DF
13983 && hmR0VmxIsPinCtlsSet(pVCpu, pVmxTransient, VMX_PIN_CTLS_VIRT_NMI))
13984 {
13985 /*
13986 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
13987 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
13988 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
13989 */
13990 CPUMSetGuestNmiBlocking(pVCpu, true);
13991 Log4Func(("Set NMI blocking. fValid=%RTbool uExitReason=%u\n", VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo),
13992 pVmxTransient->uExitReason));
13993 }
13994
13995 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
13996 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
13997 return rcStrict;
13998}
13999
14000
14001/** @name VM-exit handlers.
14002 * @{
14003 */
14004/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14005/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
14006/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14007
14008/**
14009 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
14010 */
14011HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14012{
14013 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14014 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
14015 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
14016 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
14017 return VINF_SUCCESS;
14018 return VINF_EM_RAW_INTERRUPT;
14019}
14020
14021
14022/**
14023 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI). Conditional
14024 * VM-exit.
14025 */
14026HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14027{
14028 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14029 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
14030
14031 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14032 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14033 AssertRCReturn(rc, rc);
14034
14035 uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
14036 Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
14037 && uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
14038 Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
14039
14040 if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
14041 {
14042 /*
14043 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
14044 * injected it ourselves and anything we inject is not going to cause a VM-exit directly
14045 * for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
14046 *
14047 * [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
14048 * [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
14049 */
14050 return hmR0VmxExitHostNmi(pVCpu);
14051 }
14052
14053 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14054 VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14055 if (RT_UNLIKELY(rcStrict == VINF_SUCCESS))
14056 { /* likely */ }
14057 else
14058 {
14059 if (rcStrict == VINF_HM_DOUBLE_FAULT)
14060 rcStrict = VINF_SUCCESS;
14061 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
14062 return rcStrict;
14063 }
14064
14065 uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
14066 uint32_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
14067 switch (uIntType)
14068 {
14069 case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
14070 Assert(uVector == X86_XCPT_DB);
14071 RT_FALL_THRU();
14072 case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
14073 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
14074 RT_FALL_THRU();
14075 case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
14076 {
14077 /*
14078 * If there's any exception caused as a result of event injection, the resulting
14079 * secondary/final execption will be pending, we shall continue guest execution
14080 * after injecting the event. The page-fault case is complicated and we manually
14081 * handle any currently pending event in hmR0VmxExitXcptPF.
14082 */
14083 if (!pVCpu->hm.s.Event.fPending)
14084 { /* likely */ }
14085 else if (uVector != X86_XCPT_PF)
14086 {
14087 rcStrict = VINF_SUCCESS;
14088 break;
14089 }
14090
14091 switch (uVector)
14092 {
14093 case X86_XCPT_PF: rcStrict = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
14094 case X86_XCPT_GP: rcStrict = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
14095 case X86_XCPT_MF: rcStrict = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
14096 case X86_XCPT_DB: rcStrict = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
14097 case X86_XCPT_BP: rcStrict = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
14098 case X86_XCPT_AC: rcStrict = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
14099
14100 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
14101 rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
14102 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
14103 rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
14104 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
14105 rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
14106 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
14107 rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
14108 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
14109 rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
14110 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
14111 rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
14112 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
14113 rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
14114 default:
14115 {
14116 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
14117 if (pVmcsInfo->RealMode.fRealOnV86Active)
14118 {
14119 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
14120 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
14121 Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
14122
14123 rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
14124 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14125 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14126 AssertRCReturn(rc, rc);
14127 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
14128 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
14129 0 /* GCPtrFaultAddress */);
14130 rcStrict = VINF_SUCCESS;
14131 }
14132 else
14133 {
14134 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
14135 pVCpu->hm.s.u32HMError = uVector;
14136 rcStrict = VERR_VMX_UNEXPECTED_EXCEPTION;
14137 }
14138 break;
14139 }
14140 }
14141 break;
14142 }
14143
14144 default:
14145 {
14146 pVCpu->hm.s.u32HMError = uExitIntInfo;
14147 rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
14148 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
14149 break;
14150 }
14151 }
14152 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
14153 return rcStrict;
14154}
14155
14156
14157/**
14158 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
14159 */
14160HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14161{
14162 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14163
14164 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
14165 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14166 int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
14167 AssertRCReturn(rc, rc);
14168
14169 /* Evaluate and deliver pending events and resume guest execution. */
14170 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
14171 return VINF_SUCCESS;
14172}
14173
14174
14175/**
14176 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
14177 */
14178HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14179{
14180 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14181
14182 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14183 if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
14184 {
14185 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
14186 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14187 }
14188
14189 Assert(!CPUMIsGuestNmiBlocking(pVCpu));
14190
14191 /*
14192 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
14193 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
14194 */
14195 uint32_t fIntrState;
14196 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
14197 AssertRCReturn(rc, rc);
14198 Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
14199 if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
14200 {
14201 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
14202 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
14203
14204 fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
14205 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
14206 AssertRCReturn(rc, rc);
14207 }
14208
14209 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
14210 rc = hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
14211 AssertRCReturn(rc, rc);
14212
14213 /* Evaluate and deliver pending events and resume guest execution. */
14214 return VINF_SUCCESS;
14215}
14216
14217
14218/**
14219 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
14220 */
14221HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14222{
14223 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14224 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14225}
14226
14227
14228/**
14229 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
14230 */
14231HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14232{
14233 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14234 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14235}
14236
14237
14238/**
14239 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
14240 */
14241HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14242{
14243 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14244
14245 /*
14246 * Get the state we need and update the exit history entry.
14247 */
14248 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14249 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14250 rc |= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
14251 AssertRCReturn(rc, rc);
14252
14253 VBOXSTRICTRC rcStrict;
14254 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14255 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
14256 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14257 if (!pExitRec)
14258 {
14259 /*
14260 * Regular CPUID instruction execution.
14261 */
14262 rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
14263 if (rcStrict == VINF_SUCCESS)
14264 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
14265 else if (rcStrict == VINF_IEM_RAISED_XCPT)
14266 {
14267 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14268 rcStrict = VINF_SUCCESS;
14269 }
14270 }
14271 else
14272 {
14273 /*
14274 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
14275 */
14276 int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14277 AssertRCReturn(rc2, rc2);
14278
14279 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
14280 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
14281
14282 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14283 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14284
14285 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14286 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14287 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14288 }
14289 return rcStrict;
14290}
14291
14292
14293/**
14294 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
14295 */
14296HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14297{
14298 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14299
14300 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14301 int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
14302 AssertRCReturn(rc, rc);
14303
14304 if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
14305 return VINF_EM_RAW_EMULATE_INSTR;
14306
14307 AssertMsgFailed(("hmR0VmxExitGetsec: Unexpected VM-exit when CR4.SMXE is 0.\n"));
14308 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14309}
14310
14311
14312/**
14313 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
14314 */
14315HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14316{
14317 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14318
14319 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14320 int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14321 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14322 AssertRCReturn(rc, rc);
14323
14324 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
14325 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14326 {
14327 /* If we get a spurious VM-exit when TSC offsetting is enabled,
14328 we must reset offsetting on VM-entry. See @bugref{6634}. */
14329 if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
14330 pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14331 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
14332 }
14333 else if (rcStrict == VINF_IEM_RAISED_XCPT)
14334 {
14335 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14336 rcStrict = VINF_SUCCESS;
14337 }
14338 return rcStrict;
14339}
14340
14341
14342/**
14343 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
14344 */
14345HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14346{
14347 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14348
14349 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14350 int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_TSC_AUX);
14351 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14352 AssertRCReturn(rc, rc);
14353
14354 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
14355 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14356 {
14357 /* If we get a spurious VM-exit when TSC offsetting is enabled,
14358 we must reset offsetting on VM-reentry. See @bugref{6634}. */
14359 if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
14360 pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14361 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
14362 }
14363 else if (rcStrict == VINF_IEM_RAISED_XCPT)
14364 {
14365 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14366 rcStrict = VINF_SUCCESS;
14367 }
14368 return rcStrict;
14369}
14370
14371
14372/**
14373 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
14374 */
14375HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14376{
14377 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14378
14379 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14380 int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR0
14381 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
14382 AssertRCReturn(rc, rc);
14383
14384 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14385 rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
14386 if (RT_LIKELY(rc == VINF_SUCCESS))
14387 {
14388 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14389 Assert(pVmxTransient->cbInstr == 2);
14390 }
14391 else
14392 {
14393 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
14394 rc = VERR_EM_INTERPRETER;
14395 }
14396 return rc;
14397}
14398
14399
14400/**
14401 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
14402 */
14403HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14404{
14405 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14406
14407 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
14408 if (EMAreHypercallInstructionsEnabled(pVCpu))
14409 {
14410 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14411 int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CR0
14412 | CPUMCTX_EXTRN_SS | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_EFER);
14413 AssertRCReturn(rc, rc);
14414
14415 /* Perform the hypercall. */
14416 rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
14417 if (rcStrict == VINF_SUCCESS)
14418 {
14419 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14420 AssertRCReturn(rc, rc);
14421 }
14422 else
14423 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
14424 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
14425 || RT_FAILURE(rcStrict));
14426
14427 /* If the hypercall changes anything other than guest's general-purpose registers,
14428 we would need to reload the guest changed bits here before VM-entry. */
14429 }
14430 else
14431 Log4Func(("Hypercalls not enabled\n"));
14432
14433 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
14434 if (RT_FAILURE(rcStrict))
14435 {
14436 hmR0VmxSetPendingXcptUD(pVCpu);
14437 rcStrict = VINF_SUCCESS;
14438 }
14439
14440 return rcStrict;
14441}
14442
14443
14444/**
14445 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
14446 */
14447HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14448{
14449 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14450 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
14451
14452 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14453 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14454 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14455 rc |= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
14456 AssertRCReturn(rc, rc);
14457
14458 VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
14459
14460 if (rcStrict == VINF_SUCCESS || rcStrict == VINF_PGM_SYNC_CR3)
14461 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
14462 else if (rcStrict == VINF_IEM_RAISED_XCPT)
14463 {
14464 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14465 rcStrict = VINF_SUCCESS;
14466 }
14467 else
14468 AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
14469 VBOXSTRICTRC_VAL(rcStrict)));
14470 return rcStrict;
14471}
14472
14473
14474/**
14475 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
14476 */
14477HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14478{
14479 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14480
14481 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14482 int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_DS);
14483 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14484 AssertRCReturn(rc, rc);
14485
14486 VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbInstr);
14487 if (rcStrict == VINF_SUCCESS)
14488 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
14489 else if (rcStrict == VINF_IEM_RAISED_XCPT)
14490 {
14491 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14492 rcStrict = VINF_SUCCESS;
14493 }
14494
14495 return rcStrict;
14496}
14497
14498
14499/**
14500 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
14501 */
14502HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14503{
14504 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14505
14506 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14507 int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
14508 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14509 AssertRCReturn(rc, rc);
14510
14511 VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbInstr);
14512 if (RT_SUCCESS(rcStrict))
14513 {
14514 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
14515 if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
14516 rcStrict = VINF_SUCCESS;
14517 }
14518
14519 return rcStrict;
14520}
14521
14522
14523/**
14524 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
14525 * VM-exit.
14526 */
14527HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14528{
14529 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14530 return VINF_EM_RESET;
14531}
14532
14533
14534/**
14535 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
14536 */
14537HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14538{
14539 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14540
14541 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14542 AssertRCReturn(rc, rc);
14543
14544 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
14545 if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
14546 rc = VINF_SUCCESS;
14547 else
14548 rc = VINF_EM_HALT;
14549
14550 if (rc != VINF_SUCCESS)
14551 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
14552 return rc;
14553}
14554
14555
14556/**
14557 * VM-exit handler for instructions that result in a \#UD exception delivered to
14558 * the guest.
14559 */
14560HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14561{
14562 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14563 hmR0VmxSetPendingXcptUD(pVCpu);
14564 return VINF_SUCCESS;
14565}
14566
14567
14568/**
14569 * VM-exit handler for expiry of the VMX-preemption timer.
14570 */
14571HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14572{
14573 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14574
14575 /* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
14576 pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14577
14578 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
14579 PVM pVM = pVCpu->CTX_SUFF(pVM);
14580 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
14581 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
14582 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
14583}
14584
14585
14586/**
14587 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
14588 */
14589HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14590{
14591 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14592
14593 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14594 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14595 rc |= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_CR4);
14596 AssertRCReturn(rc, rc);
14597
14598 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
14599 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
14600 : HM_CHANGED_RAISED_XCPT_MASK);
14601
14602 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14603 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
14604
14605 return rcStrict;
14606}
14607
14608
14609/**
14610 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
14611 */
14612HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14613{
14614 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14615 /** @todo Use VM-exit instruction information. */
14616 return VERR_EM_INTERPRETER;
14617}
14618
14619
14620/**
14621 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error
14622 * VM-exit.
14623 */
14624HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14625{
14626 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14627 int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14628 AssertRCReturn(rc, rc);
14629
14630 rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo);
14631 if (RT_FAILURE(rc))
14632 return rc;
14633
14634 uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
14635 NOREF(uInvalidReason);
14636
14637#ifdef VBOX_STRICT
14638 uint32_t fIntrState;
14639 RTHCUINTREG uHCReg;
14640 uint64_t u64Val;
14641 uint32_t u32Val;
14642 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
14643 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
14644 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
14645 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
14646 AssertRCReturn(rc, rc);
14647
14648 Log4(("uInvalidReason %u\n", uInvalidReason));
14649 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
14650 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
14651 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
14652 Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
14653
14654 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
14655 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
14656 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
14657 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
14658 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
14659 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
14660 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
14661 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
14662 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
14663 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
14664 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
14665 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
14666
14667 hmR0DumpRegs(pVCpu);
14668#endif
14669
14670 return VERR_VMX_INVALID_GUEST_STATE;
14671}
14672
14673/**
14674 * VM-exit handler for all undefined/unexpected reasons. Should never happen.
14675 */
14676HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUnexpected(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14677{
14678 /*
14679 * Cummulative notes of all recognized but unexpected VM-exits.
14680 *
14681 * 1. This does -not- cover scenarios like like a page-fault VM-exit occurring when
14682 * nested-paging is used.
14683 *
14684 * 2. Any instruction that causes a VM-exit unconditionally (for e.g. VMXON) must be
14685 * emulated or a #UD must be raised in the guest. Therefore, we should -not- be using
14686 * this function (and thereby stop VM execution) for handling such instructions.
14687 *
14688 *
14689 * VMX_EXIT_INIT_SIGNAL:
14690 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
14691 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
14692 * VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
14693 *
14694 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
14695 * See Intel spec. 29.3 "VMX Instructions" for "VMXON".
14696 * See Intel spec. "23.8 Restrictions on VMX operation".
14697 *
14698 * VMX_EXIT_SIPI:
14699 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
14700 * activity state is used. We don't make use of it as our guests don't have direct
14701 * access to the host local APIC.
14702 *
14703 * See Intel spec. 25.3 "Other Causes of VM-exits".
14704 *
14705 * VMX_EXIT_IO_SMI:
14706 * VMX_EXIT_SMI:
14707 * This can only happen if we support dual-monitor treatment of SMI, which can be
14708 * activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
14709 * monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
14710 * VMX root mode or receive an SMI. If we get here, something funny is going on.
14711 *
14712 * See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
14713 * See Intel spec. 25.3 "Other Causes of VM-Exits"
14714 *
14715 * VMX_EXIT_ERR_MSR_LOAD:
14716 * Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
14717 * and typically indicates a bug in the hypervisor code. We thus cannot not resume
14718 * execution.
14719 *
14720 * See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
14721 *
14722 * VMX_EXIT_ERR_MACHINE_CHECK:
14723 * Machine check exceptions indicates a fatal/unrecoverable hardware condition
14724 * including but not limited to system bus, ECC, parity, cache and TLB errors. A
14725 * #MC exception abort class exception is raised. We thus cannot assume a
14726 * reasonable chance of continuing any sort of execution and we bail.
14727 *
14728 * See Intel spec. 15.1 "Machine-check Architecture".
14729 * See Intel spec. 27.1 "Architectural State Before A VM Exit".
14730 *
14731 * VMX_EXIT_PML_FULL:
14732 * VMX_EXIT_VIRTUALIZED_EOI:
14733 * VMX_EXIT_APIC_WRITE:
14734 * We do not currently support any of these features and thus they are all unexpected
14735 * VM-exits.
14736 *
14737 * VMX_EXIT_GDTR_IDTR_ACCESS:
14738 * VMX_EXIT_LDTR_TR_ACCESS:
14739 * VMX_EXIT_RDRAND:
14740 * VMX_EXIT_RSM:
14741 * VMX_EXIT_VMFUNC:
14742 * VMX_EXIT_ENCLS:
14743 * VMX_EXIT_RDSEED:
14744 * VMX_EXIT_XSAVES:
14745 * VMX_EXIT_XRSTORS:
14746 * VMX_EXIT_UMWAIT:
14747 * VMX_EXIT_TPAUSE:
14748 * These VM-exits are -not- caused unconditionally by execution of the corresponding
14749 * instruction. Any VM-exit for these instructions indicate a hardware problem,
14750 * unsupported CPU modes (like SMM) or potentially corrupt VMCS controls.
14751 *
14752 * See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
14753 */
14754 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14755 AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
14756 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14757}
14758
14759
14760/**
14761 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
14762 */
14763HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14764{
14765 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14766
14767 /** @todo Optimize this: We currently drag in in the whole MSR state
14768 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
14769 * MSRs required. That would require changes to IEM and possibly CPUM too.
14770 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
14771 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14772 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
14773 uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS;
14774 switch (idMsr)
14775 {
14776 case MSR_K8_FS_BASE: fImport |= CPUMCTX_EXTRN_FS; break;
14777 case MSR_K8_GS_BASE: fImport |= CPUMCTX_EXTRN_GS; break;
14778 }
14779
14780 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14781 rc |= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
14782 AssertRCReturn(rc, rc);
14783
14784 Log4Func(("ecx=%#RX32\n", idMsr));
14785
14786#ifdef VBOX_STRICT
14787 if (hmR0VmxIsProcCtlsSet(pVCpu, pVmxTransient, VMX_PROC_CTLS_USE_MSR_BITMAPS))
14788 {
14789 if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
14790 && idMsr != MSR_K6_EFER)
14791 {
14792 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
14793 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14794 }
14795 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14796 {
14797 Assert(pVmcsInfo->pvMsrBitmap);
14798 uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
14799 if (fMsrpm & VMXMSRPM_ALLOW_RD)
14800 {
14801 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
14802 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14803 }
14804 }
14805 }
14806#endif
14807
14808 VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
14809 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
14810 if (rcStrict == VINF_SUCCESS)
14811 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
14812 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX);
14813 else if (rcStrict == VINF_IEM_RAISED_XCPT)
14814 {
14815 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14816 rcStrict = VINF_SUCCESS;
14817 }
14818 else
14819 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
14820
14821 return rcStrict;
14822}
14823
14824
14825/**
14826 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
14827 */
14828HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14829{
14830 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14831
14832 /** @todo Optimize this: We currently drag in in the whole MSR state
14833 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
14834 * MSRs required. That would require changes to IEM and possibly CPUM too.
14835 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
14836 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
14837 uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS;
14838
14839 /*
14840 * The FS and GS base MSRs are not part of the above all-MSRs mask.
14841 * Although we don't need to fetch the base as it will be overwritten shortly, while
14842 * loading guest-state we would also load the entire segment register including limit
14843 * and attributes and thus we need to load them here.
14844 */
14845 switch (idMsr)
14846 {
14847 case MSR_K8_FS_BASE: fImport |= CPUMCTX_EXTRN_FS; break;
14848 case MSR_K8_GS_BASE: fImport |= CPUMCTX_EXTRN_GS; break;
14849 }
14850
14851 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14852 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14853 rc |= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
14854 AssertRCReturn(rc, rc);
14855
14856 Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
14857
14858 VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
14859 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
14860
14861 if (rcStrict == VINF_SUCCESS)
14862 {
14863 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
14864
14865 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
14866 if ( idMsr == MSR_IA32_APICBASE
14867 || ( idMsr >= MSR_IA32_X2APIC_START
14868 && idMsr <= MSR_IA32_X2APIC_END))
14869 {
14870 /*
14871 * We've already saved the APIC related guest-state (TPR) in post-run phase.
14872 * When full APIC register virtualization is implemented we'll have to make
14873 * sure APIC state is saved from the VMCS before IEM changes it.
14874 */
14875 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
14876 }
14877 else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
14878 pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14879 else if (idMsr == MSR_K6_EFER)
14880 {
14881 /*
14882 * If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
14883 * as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
14884 * We care about the other bits as well, SCE and NXE. See @bugref{7368}.
14885 */
14886 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
14887 }
14888
14889 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
14890 if (!hmR0VmxIsProcCtlsSet(pVCpu, pVmxTransient, VMX_PROC_CTLS_USE_MSR_BITMAPS))
14891 {
14892 switch (idMsr)
14893 {
14894 case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
14895 case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
14896 case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
14897 case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
14898 case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
14899 case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
14900 default:
14901 {
14902 if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
14903 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
14904 else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14905 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
14906 break;
14907 }
14908 }
14909 }
14910#ifdef VBOX_STRICT
14911 else
14912 {
14913 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
14914 switch (idMsr)
14915 {
14916 case MSR_IA32_SYSENTER_CS:
14917 case MSR_IA32_SYSENTER_EIP:
14918 case MSR_IA32_SYSENTER_ESP:
14919 case MSR_K8_FS_BASE:
14920 case MSR_K8_GS_BASE:
14921 {
14922 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
14923 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14924 }
14925
14926 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
14927 default:
14928 {
14929 if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
14930 {
14931 /* EFER MSR writes are always intercepted. */
14932 if (idMsr != MSR_K6_EFER)
14933 {
14934 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
14935 idMsr));
14936 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14937 }
14938 }
14939
14940 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14941 {
14942 Assert(pVmcsInfo->pvMsrBitmap);
14943 uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
14944 if (fMsrpm & VMXMSRPM_ALLOW_WR)
14945 {
14946 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
14947 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14948 }
14949 }
14950 break;
14951 }
14952 }
14953 }
14954#endif /* VBOX_STRICT */
14955 }
14956 else if (rcStrict == VINF_IEM_RAISED_XCPT)
14957 {
14958 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14959 rcStrict = VINF_SUCCESS;
14960 }
14961 else
14962 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
14963
14964 return rcStrict;
14965}
14966
14967
14968/**
14969 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
14970 */
14971HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14972{
14973 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14974
14975 /** @todo The guest has likely hit a contended spinlock. We might want to
14976 * poke a schedule different guest VCPU. */
14977 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14978 if (RT_SUCCESS(rc))
14979 return VINF_EM_RAW_INTERRUPT;
14980
14981 AssertMsgFailed(("hmR0VmxExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
14982 return rc;
14983}
14984
14985
14986/**
14987 * VM-exit handler for when the TPR value is lowered below the specified
14988 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
14989 */
14990HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14991{
14992 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14993 Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
14994
14995 /*
14996 * The TPR shadow would've been synced with the APIC TPR in the post-run phase.
14997 * We'll re-evaluate pending interrupts and inject them before the next VM
14998 * entry so we can just continue execution here.
14999 */
15000 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
15001 return VINF_SUCCESS;
15002}
15003
15004
15005/**
15006 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
15007 * VM-exit.
15008 *
15009 * @retval VINF_SUCCESS when guest execution can continue.
15010 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
15011 * @retval VERR_EM_RESCHEDULE_REM when we need to return to ring-3 due to
15012 * incompatible guest state for VMX execution (real-on-v86 case).
15013 */
15014HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15015{
15016 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15017 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
15018
15019 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15020 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15021 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15022 AssertRCReturn(rc, rc);
15023
15024 VBOXSTRICTRC rcStrict;
15025 PVM pVM = pVCpu->CTX_SUFF(pVM);
15026 uint64_t const uExitQual = pVmxTransient->uExitQual;
15027 uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
15028 switch (uAccessType)
15029 {
15030 /*
15031 * MOV to CRx.
15032 */
15033 case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
15034 {
15035 rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15036 AssertRCReturn(rc, rc);
15037
15038 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
15039 uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
15040 uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15041 uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15042
15043 /*
15044 * MOV to CR3 only cause a VM-exit when one or more of the following are true:
15045 * - When nested paging isn't used.
15046 * - If the guest doesn't have paging enabled (intercept CR3 to update shadow page tables).
15047 * - We are executing in the VM debug loop.
15048 */
15049 Assert( iCrReg != 3
15050 || !pVM->hm.s.fNestedPaging
15051 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15052 || pVCpu->hm.s.fUsingDebugLoop);
15053
15054 /* MOV to CR8 writes only cause VM-exits when TPR shadow is not used. */
15055 Assert( iCrReg != 8
15056 || !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15057
15058 rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
15059 AssertMsg( rcStrict == VINF_SUCCESS
15060 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15061
15062 /*
15063 * This is a kludge for handling switches back to real mode when we try to use
15064 * V86 mode to run real mode code directly. Problem is that V86 mode cannot
15065 * deal with special selector values, so we have to return to ring-3 and run
15066 * there till the selector values are V86 mode compatible.
15067 *
15068 * Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
15069 * latter is an alias for VINF_IEM_RAISED_XCPT which is asserted at the end of
15070 * this function.
15071 */
15072 if ( iCrReg == 0
15073 && rcStrict == VINF_SUCCESS
15074 && !pVM->hm.s.vmx.fUnrestrictedGuest
15075 && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
15076 && (uOldCr0 & X86_CR0_PE)
15077 && !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
15078 {
15079 /** @todo Check selectors rather than returning all the time. */
15080 Assert(!pVmxTransient->fIsNestedGuest);
15081 Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
15082 rcStrict = VINF_EM_RESCHEDULE_REM;
15083 }
15084 break;
15085 }
15086
15087 /*
15088 * MOV from CRx.
15089 */
15090 case VMX_EXIT_QUAL_CRX_ACCESS_READ:
15091 {
15092 uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15093 uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15094
15095 /*
15096 * MOV from CR3 only cause a VM-exit when one or more of the following are true:
15097 * - When nested paging isn't used.
15098 * - If the guest doesn't have paging enabled (pass guest's CR3 rather than our identity mapped CR3).
15099 * - We are executing in the VM debug loop.
15100 */
15101 Assert( iCrReg != 3
15102 || !pVM->hm.s.fNestedPaging
15103 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15104 || pVCpu->hm.s.fUsingDebugLoop);
15105
15106 /* MOV from CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
15107 Assert( iCrReg != 8
15108 || !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15109
15110 rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
15111 break;
15112 }
15113
15114 /*
15115 * CLTS (Clear Task-Switch Flag in CR0).
15116 */
15117 case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
15118 {
15119 rcStrict = hmR0VmxExitClts(pVCpu, pVmcsInfo, pVmxTransient->cbInstr);
15120 break;
15121 }
15122
15123 /*
15124 * LMSW (Load Machine-Status Word into CR0).
15125 * LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here.
15126 */
15127 case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:
15128 {
15129 RTGCPTR GCPtrEffDst;
15130 uint8_t const cbInstr = pVmxTransient->cbInstr;
15131 uint16_t const uMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual);
15132 bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual);
15133 if (fMemOperand)
15134 {
15135 rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
15136 AssertRCReturn(rc, rc);
15137 GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
15138 }
15139 else
15140 GCPtrEffDst = NIL_RTGCPTR;
15141 rcStrict = hmR0VmxExitLmsw(pVCpu, pVmcsInfo, cbInstr, uMsw, GCPtrEffDst);
15142 break;
15143 }
15144
15145 default:
15146 {
15147 AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
15148 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
15149 }
15150 }
15151
15152 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS))
15153 == (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS));
15154 Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15155
15156 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
15157 NOREF(pVM);
15158 return rcStrict;
15159}
15160
15161
15162/**
15163 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
15164 * VM-exit.
15165 */
15166HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15167{
15168 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15169 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
15170
15171 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15172 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15173 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15174 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15175 rc |= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_SREG_MASK
15176 | CPUMCTX_EXTRN_EFER);
15177 /* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
15178 AssertRCReturn(rc, rc);
15179
15180 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
15181 uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
15182 uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
15183 bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
15184 bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
15185 bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
15186 bool const fDbgStepping = pVCpu->hm.s.fSingleInstruction;
15187 AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
15188
15189 /*
15190 * Update exit history to see if this exit can be optimized.
15191 */
15192 VBOXSTRICTRC rcStrict;
15193 PCEMEXITREC pExitRec = NULL;
15194 if ( !fGstStepping
15195 && !fDbgStepping)
15196 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
15197 !fIOString
15198 ? !fIOWrite
15199 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
15200 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
15201 : !fIOWrite
15202 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
15203 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
15204 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
15205 if (!pExitRec)
15206 {
15207 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
15208 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
15209
15210 uint32_t const cbValue = s_aIOSizes[uIOSize];
15211 uint32_t const cbInstr = pVmxTransient->cbInstr;
15212 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
15213 PVM pVM = pVCpu->CTX_SUFF(pVM);
15214 if (fIOString)
15215 {
15216 /*
15217 * INS/OUTS - I/O String instruction.
15218 *
15219 * Use instruction-information if available, otherwise fall back on
15220 * interpreting the instruction.
15221 */
15222 Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
15223 AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
15224 bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
15225 if (fInsOutsInfo)
15226 {
15227 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15228 AssertRCReturn(rc2, rc2);
15229 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
15230 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
15231 IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
15232 bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
15233 if (fIOWrite)
15234 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
15235 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
15236 else
15237 {
15238 /*
15239 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
15240 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
15241 * See Intel Instruction spec. for "INS".
15242 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
15243 */
15244 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
15245 }
15246 }
15247 else
15248 rcStrict = IEMExecOne(pVCpu);
15249
15250 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
15251 fUpdateRipAlready = true;
15252 }
15253 else
15254 {
15255 /*
15256 * IN/OUT - I/O instruction.
15257 */
15258 Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
15259 uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
15260 Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
15261 if (fIOWrite)
15262 {
15263 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
15264 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
15265 if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
15266 && !pCtx->eflags.Bits.u1TF)
15267 rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
15268 }
15269 else
15270 {
15271 uint32_t u32Result = 0;
15272 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
15273 if (IOM_SUCCESS(rcStrict))
15274 {
15275 /* Save result of I/O IN instr. in AL/AX/EAX. */
15276 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Result & uAndVal);
15277 }
15278 if ( rcStrict == VINF_IOM_R3_IOPORT_READ
15279 && !pCtx->eflags.Bits.u1TF)
15280 rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
15281 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
15282 }
15283 }
15284
15285 if (IOM_SUCCESS(rcStrict))
15286 {
15287 if (!fUpdateRipAlready)
15288 {
15289 hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
15290 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
15291 }
15292
15293 /*
15294 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
15295 * while booting Fedora 17 64-bit guest.
15296 *
15297 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
15298 */
15299 if (fIOString)
15300 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
15301
15302 /*
15303 * If any I/O breakpoints are armed, we need to check if one triggered
15304 * and take appropriate action.
15305 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
15306 */
15307 rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
15308 AssertRCReturn(rc, rc);
15309
15310 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
15311 * execution engines about whether hyper BPs and such are pending. */
15312 uint32_t const uDr7 = pCtx->dr[7];
15313 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
15314 && X86_DR7_ANY_RW_IO(uDr7)
15315 && (pCtx->cr4 & X86_CR4_DE))
15316 || DBGFBpIsHwIoArmed(pVM)))
15317 {
15318 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
15319
15320 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
15321 VMMRZCallRing3Disable(pVCpu);
15322 HM_DISABLE_PREEMPT(pVCpu);
15323
15324 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
15325
15326 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
15327 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
15328 {
15329 /* Raise #DB. */
15330 if (fIsGuestDbgActive)
15331 ASMSetDR6(pCtx->dr[6]);
15332 if (pCtx->dr[7] != uDr7)
15333 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR7;
15334
15335 hmR0VmxSetPendingXcptDB(pVCpu);
15336 }
15337 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
15338 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
15339 else if ( rcStrict2 != VINF_SUCCESS
15340 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
15341 rcStrict = rcStrict2;
15342 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
15343
15344 HM_RESTORE_PREEMPT();
15345 VMMRZCallRing3Enable(pVCpu);
15346 }
15347 }
15348
15349#ifdef VBOX_STRICT
15350 if ( rcStrict == VINF_IOM_R3_IOPORT_READ
15351 || rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
15352 Assert(!fIOWrite);
15353 else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
15354 || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
15355 || rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
15356 Assert(fIOWrite);
15357 else
15358 {
15359# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
15360 * statuses, that the VMM device and some others may return. See
15361 * IOM_SUCCESS() for guidance. */
15362 AssertMsg( RT_FAILURE(rcStrict)
15363 || rcStrict == VINF_SUCCESS
15364 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
15365 || rcStrict == VINF_EM_DBG_BREAKPOINT
15366 || rcStrict == VINF_EM_RAW_GUEST_TRAP
15367 || rcStrict == VINF_EM_RAW_TO_R3
15368 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15369# endif
15370 }
15371#endif
15372 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
15373 }
15374 else
15375 {
15376 /*
15377 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
15378 */
15379 int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15380 AssertRCReturn(rc2, rc2);
15381 STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
15382 : fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
15383 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
15384 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15385 VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
15386 fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
15387
15388 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
15389 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15390
15391 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
15392 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15393 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
15394 }
15395 return rcStrict;
15396}
15397
15398
15399/**
15400 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
15401 * VM-exit.
15402 */
15403HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15404{
15405 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15406
15407 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
15408 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15409 AssertRCReturn(rc, rc);
15410 if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
15411 {
15412 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15413 AssertRCReturn(rc, rc);
15414 if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
15415 {
15416 uint32_t uErrCode;
15417 RTGCUINTPTR GCPtrFaultAddress;
15418 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
15419 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
15420 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
15421 if (fErrorCodeValid)
15422 {
15423 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15424 AssertRCReturn(rc, rc);
15425 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
15426 }
15427 else
15428 uErrCode = 0;
15429
15430 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
15431 && uVector == X86_XCPT_PF)
15432 GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
15433 else
15434 GCPtrFaultAddress = 0;
15435
15436 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15437 AssertRCReturn(rc, rc);
15438
15439 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
15440 pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
15441
15442 Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
15443 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
15444 return VINF_EM_RAW_INJECT_TRPM_EVENT;
15445 }
15446 }
15447
15448 /* Fall back to the interpreter to emulate the task-switch. */
15449 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
15450 return VERR_EM_INTERPRETER;
15451}
15452
15453
15454/**
15455 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
15456 */
15457HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15458{
15459 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15460
15461 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15462 pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
15463 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
15464 AssertRCReturn(rc, rc);
15465 return VINF_EM_DBG_STEPPED;
15466}
15467
15468
15469/**
15470 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
15471 */
15472HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15473{
15474 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15475 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
15476
15477 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
15478 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15479 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
15480 {
15481 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
15482 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
15483 {
15484 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
15485 return VINF_EM_RAW_INJECT_TRPM_EVENT;
15486 }
15487 }
15488 else
15489 {
15490 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
15491 rcStrict1 = VINF_SUCCESS;
15492 return rcStrict1;
15493 }
15494
15495 /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
15496 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15497 int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15498 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15499 AssertRCReturn(rc, rc);
15500
15501 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
15502 uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
15503 VBOXSTRICTRC rcStrict2;
15504 switch (uAccessType)
15505 {
15506 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
15507 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
15508 {
15509 AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
15510 || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
15511 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
15512
15513 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
15514 GCPhys &= PAGE_BASE_GC_MASK;
15515 GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
15516 PVM pVM = pVCpu->CTX_SUFF(pVM);
15517 Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
15518 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
15519
15520 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15521 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
15522 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
15523 CPUMCTX2CORE(pCtx), GCPhys);
15524 Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
15525 if ( rcStrict2 == VINF_SUCCESS
15526 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
15527 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
15528 {
15529 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
15530 | HM_CHANGED_GUEST_APIC_TPR);
15531 rcStrict2 = VINF_SUCCESS;
15532 }
15533 break;
15534 }
15535
15536 default:
15537 Log4Func(("uAccessType=%#x\n", uAccessType));
15538 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
15539 break;
15540 }
15541
15542 if (rcStrict2 != VINF_SUCCESS)
15543 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
15544 return rcStrict2;
15545}
15546
15547
15548/**
15549 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
15550 * VM-exit.
15551 */
15552HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15553{
15554 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15555
15556 /* We should -not- get this VM-exit if the guest's debug registers were active. */
15557 if (pVmxTransient->fWasGuestDebugStateActive)
15558 {
15559 AssertMsgFailed(("Unexpected MOV DRx exit\n"));
15560 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
15561 }
15562
15563 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15564 if ( !pVCpu->hm.s.fSingleInstruction
15565 && !pVmxTransient->fWasHyperDebugStateActive)
15566 {
15567 Assert(!DBGFIsStepping(pVCpu));
15568 Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
15569
15570 /* Don't intercept MOV DRx any more. */
15571 pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
15572 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
15573 AssertRCReturn(rc, rc);
15574
15575 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
15576 VMMRZCallRing3Disable(pVCpu);
15577 HM_DISABLE_PREEMPT(pVCpu);
15578
15579 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
15580 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
15581 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
15582
15583 HM_RESTORE_PREEMPT();
15584 VMMRZCallRing3Enable(pVCpu);
15585
15586#ifdef VBOX_WITH_STATISTICS
15587 rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15588 AssertRCReturn(rc, rc);
15589 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
15590 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
15591 else
15592 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
15593#endif
15594 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
15595 return VINF_SUCCESS;
15596 }
15597
15598 /*
15599 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
15600 * The EFER MSR is always up-to-date.
15601 * Update the segment registers and DR7 from the CPU.
15602 */
15603 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15604 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15605 rc |= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_DR7);
15606 AssertRCReturn(rc, rc);
15607 Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
15608
15609 PVM pVM = pVCpu->CTX_SUFF(pVM);
15610 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
15611 {
15612 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
15613 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
15614 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
15615 if (RT_SUCCESS(rc))
15616 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
15617 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
15618 }
15619 else
15620 {
15621 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
15622 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
15623 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
15624 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
15625 }
15626
15627 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
15628 if (RT_SUCCESS(rc))
15629 {
15630 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15631 AssertRCReturn(rc2, rc2);
15632 return VINF_SUCCESS;
15633 }
15634 return rc;
15635}
15636
15637
15638/**
15639 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
15640 * Conditional VM-exit.
15641 */
15642HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15643{
15644 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15645 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
15646
15647 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
15648 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15649 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
15650 {
15651 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
15652 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
15653 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
15654 {
15655 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
15656 return VINF_EM_RAW_INJECT_TRPM_EVENT;
15657 }
15658 }
15659 else
15660 {
15661 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
15662 rcStrict1 = VINF_SUCCESS;
15663 return rcStrict1;
15664 }
15665
15666 /*
15667 * Get sufficent state and update the exit history entry.
15668 */
15669 RTGCPHYS GCPhys;
15670 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15671 int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
15672 rc |= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15673 AssertRCReturn(rc, rc);
15674
15675 VBOXSTRICTRC rcStrict;
15676 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
15677 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
15678 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
15679 if (!pExitRec)
15680 {
15681 /*
15682 * If we succeed, resume guest execution.
15683 * If we fail in interpreting the instruction because we couldn't get the guest physical address
15684 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
15685 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
15686 * weird case. See @bugref{6043}.
15687 */
15688 PVM pVM = pVCpu->CTX_SUFF(pVM);
15689 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15690 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
15691 Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
15692 if ( rcStrict == VINF_SUCCESS
15693 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
15694 || rcStrict == VERR_PAGE_NOT_PRESENT)
15695 {
15696 /* Successfully handled MMIO operation. */
15697 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
15698 | HM_CHANGED_GUEST_APIC_TPR);
15699 rcStrict = VINF_SUCCESS;
15700 }
15701 }
15702 else
15703 {
15704 /*
15705 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
15706 */
15707 int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15708 AssertRCReturn(rc2, rc2);
15709
15710 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
15711 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
15712
15713 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
15714 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15715
15716 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
15717 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15718 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
15719 }
15720 return VBOXSTRICTRC_TODO(rcStrict);
15721}
15722
15723
15724/**
15725 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
15726 * VM-exit.
15727 */
15728HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15729{
15730 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15731 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
15732
15733 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
15734 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15735 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
15736 {
15737 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
15738 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
15739 Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
15740 }
15741 else
15742 {
15743 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
15744 rcStrict1 = VINF_SUCCESS;
15745 return rcStrict1;
15746 }
15747
15748 RTGCPHYS GCPhys;
15749 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15750 int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
15751 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15752 rc |= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15753 AssertRCReturn(rc, rc);
15754
15755 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
15756 AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
15757
15758 RTGCUINT uErrorCode = 0;
15759 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
15760 uErrorCode |= X86_TRAP_PF_ID;
15761 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
15762 uErrorCode |= X86_TRAP_PF_RW;
15763 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
15764 uErrorCode |= X86_TRAP_PF_P;
15765
15766 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
15767
15768
15769 /* Handle the pagefault trap for the nested shadow table. */
15770 PVM pVM = pVCpu->CTX_SUFF(pVM);
15771 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15772
15773 Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x cs:rip=%#04x:%#RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
15774 pCtx->cs.Sel, pCtx->rip));
15775
15776 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
15777 TRPMResetTrap(pVCpu);
15778
15779 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
15780 if ( rcStrict2 == VINF_SUCCESS
15781 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
15782 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
15783 {
15784 /* Successfully synced our nested page tables. */
15785 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
15786 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
15787 return VINF_SUCCESS;
15788 }
15789
15790 Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
15791 return rcStrict2;
15792}
15793
15794/** @} */
15795
15796/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15797/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
15798/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15799
15800/**
15801 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
15802 */
15803static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15804{
15805 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15806 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
15807
15808 int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
15809 AssertRCReturn(rc, rc);
15810
15811 if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
15812 {
15813 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
15814 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
15815
15816 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
15817 * provides VM-exit instruction length. If this causes problem later,
15818 * disassemble the instruction like it's done on AMD-V. */
15819 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15820 AssertRCReturn(rc2, rc2);
15821 return rc;
15822 }
15823
15824 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15825 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15826 return rc;
15827}
15828
15829
15830/**
15831 * VM-exit exception handler for \#BP (Breakpoint exception).
15832 */
15833static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15834{
15835 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15836 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
15837
15838 int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15839 AssertRCReturn(rc, rc);
15840
15841 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15842 rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
15843 if (rc == VINF_EM_RAW_GUEST_TRAP)
15844 {
15845 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15846 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15847 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15848 AssertRCReturn(rc, rc);
15849
15850 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15851 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15852 }
15853
15854 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
15855 return rc;
15856}
15857
15858
15859/**
15860 * VM-exit exception handler for \#AC (alignment check exception).
15861 */
15862static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15863{
15864 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15865
15866 /*
15867 * Re-inject it. We'll detect any nesting before getting here.
15868 */
15869 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15870 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15871 AssertRCReturn(rc, rc);
15872 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
15873
15874 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15875 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15876 return VINF_SUCCESS;
15877}
15878
15879
15880/**
15881 * VM-exit exception handler for \#DB (Debug exception).
15882 */
15883static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15884{
15885 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15886 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
15887
15888 /*
15889 * Get the DR6-like values from the Exit qualification and pass it to DBGF for processing.
15890 */
15891 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15892
15893 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
15894 uint64_t const uDR6 = X86_DR6_INIT_VAL
15895 | (pVmxTransient->uExitQual & ( X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3
15896 | X86_DR6_BD | X86_DR6_BS));
15897
15898 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15899 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
15900 Log6Func(("rc=%Rrc\n", rc));
15901 if (rc == VINF_EM_RAW_GUEST_TRAP)
15902 {
15903 /*
15904 * The exception was for the guest. Update DR6, DR7.GD and
15905 * IA32_DEBUGCTL.LBR before forwarding it.
15906 * See Intel spec. 27.1 "Architectural State before a VM-Exit".
15907 */
15908 VMMRZCallRing3Disable(pVCpu);
15909 HM_DISABLE_PREEMPT(pVCpu);
15910
15911 pCtx->dr[6] &= ~X86_DR6_B_MASK;
15912 pCtx->dr[6] |= uDR6;
15913 if (CPUMIsGuestDebugStateActive(pVCpu))
15914 ASMSetDR6(pCtx->dr[6]);
15915
15916 HM_RESTORE_PREEMPT();
15917 VMMRZCallRing3Enable(pVCpu);
15918
15919 rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
15920 AssertRCReturn(rc, rc);
15921
15922 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
15923 pCtx->dr[7] &= ~X86_DR7_GD;
15924
15925 /* Paranoia. */
15926 pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
15927 pCtx->dr[7] |= X86_DR7_RA1_MASK;
15928
15929 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
15930 AssertRCReturn(rc, rc);
15931
15932 /*
15933 * Raise #DB in the guest.
15934 *
15935 * It is important to reflect exactly what the VM-exit gave us (preserving the
15936 * interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
15937 * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
15938 * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
15939 *
15940 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
15941 * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
15942 */
15943 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15944 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15945 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15946 AssertRCReturn(rc, rc);
15947 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15948 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15949 return VINF_SUCCESS;
15950 }
15951
15952 /*
15953 * Not a guest trap, must be a hypervisor related debug event then.
15954 * Update DR6 in case someone is interested in it.
15955 */
15956 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
15957 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
15958 CPUMSetHyperDR6(pVCpu, uDR6);
15959
15960 return rc;
15961}
15962
15963
15964/**
15965 * Hacks its way around the lovely mesa driver's backdoor accesses.
15966 *
15967 * @sa hmR0SvmHandleMesaDrvGp.
15968 */
15969static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
15970{
15971 LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
15972 RT_NOREF(pCtx);
15973
15974 /* For now we'll just skip the instruction. */
15975 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15976}
15977
15978
15979/**
15980 * Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
15981 * backdoor logging w/o checking what it is running inside.
15982 *
15983 * This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
15984 * backdoor port and magic numbers loaded in registers.
15985 *
15986 * @returns true if it is, false if it isn't.
15987 * @sa hmR0SvmIsMesaDrvGp.
15988 */
15989DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
15990{
15991 /* 0xed: IN eAX,dx */
15992 uint8_t abInstr[1];
15993 if (pVmxTransient->cbInstr != sizeof(abInstr))
15994 return false;
15995
15996 /* Check that it is #GP(0). */
15997 if (pVmxTransient->uExitIntErrorCode != 0)
15998 return false;
15999
16000 /* Check magic and port. */
16001 Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_RCX)));
16002 /*Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));*/
16003 if (pCtx->rax != UINT32_C(0x564d5868))
16004 return false;
16005 if (pCtx->dx != UINT32_C(0x5658))
16006 return false;
16007
16008 /* Flat ring-3 CS. */
16009 AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
16010 Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
16011 /*Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));*/
16012 if (pCtx->cs.Attr.n.u2Dpl != 3)
16013 return false;
16014 if (pCtx->cs.u64Base != 0)
16015 return false;
16016
16017 /* Check opcode. */
16018 AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
16019 Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
16020 int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
16021 /*Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));*/
16022 if (RT_FAILURE(rc))
16023 return false;
16024 if (abInstr[0] != 0xed)
16025 return false;
16026
16027 return true;
16028}
16029
16030
16031/**
16032 * VM-exit exception handler for \#GP (General-protection exception).
16033 *
16034 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
16035 */
16036static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16037{
16038 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16039 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
16040
16041 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16042 PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16043 if (pVmcsInfo->RealMode.fRealOnV86Active)
16044 { /* likely */ }
16045 else
16046 {
16047#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
16048 Assert(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv || pVmxTransient->fIsNestedGuest);
16049#endif
16050 /* If the guest is not in real-mode or we have unrestricted guest execution support, reflect #GP to the guest. */
16051 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
16052 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
16053 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16054 rc |= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16055 AssertRCReturn(rc, rc);
16056 Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
16057 pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
16058
16059 if ( pVmxTransient->fIsNestedGuest
16060 || !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
16061 || !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
16062 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
16063 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
16064 else
16065 rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
16066 return rc;
16067 }
16068
16069 Assert(CPUMIsGuestInRealModeEx(pCtx));
16070 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
16071 Assert(!pVmxTransient->fIsNestedGuest);
16072
16073 int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16074 AssertRCReturn(rc, rc);
16075
16076 VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
16077 if (rcStrict == VINF_SUCCESS)
16078 {
16079 if (!CPUMIsGuestInRealModeEx(pCtx))
16080 {
16081 /*
16082 * The guest is no longer in real-mode, check if we can continue executing the
16083 * guest using hardware-assisted VMX. Otherwise, fall back to emulation.
16084 */
16085 pVmcsInfo->RealMode.fRealOnV86Active = false;
16086 if (HMCanExecuteVmxGuest(pVCpu, pCtx))
16087 {
16088 Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
16089 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16090 }
16091 else
16092 {
16093 Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
16094 rcStrict = VINF_EM_RESCHEDULE;
16095 }
16096 }
16097 else
16098 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16099 }
16100 else if (rcStrict == VINF_IEM_RAISED_XCPT)
16101 {
16102 rcStrict = VINF_SUCCESS;
16103 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16104 }
16105 return VBOXSTRICTRC_VAL(rcStrict);
16106}
16107
16108
16109/**
16110 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
16111 * the exception reported in the VMX transient structure back into the VM.
16112 *
16113 * @remarks Requires uExitIntInfo in the VMX transient structure to be
16114 * up-to-date.
16115 */
16116static VBOXSTRICTRC hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16117{
16118 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16119#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
16120 PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16121 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVmcsInfo->RealMode.fRealOnV86Active || pVmxTransient->fIsNestedGuest,
16122 ("uVector=%#x u32XcptBitmap=%#X32\n",
16123 VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
16124 NOREF(pVmcsInfo);
16125#endif
16126
16127 /*
16128 * Re-inject the exception into the guest. This cannot be a double-fault condition which
16129 * would have been handled while checking exits due to event delivery.
16130 */
16131 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
16132 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16133 AssertRCReturn(rc, rc);
16134 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
16135
16136#ifdef DEBUG_ramshankar
16137 rc |= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
16138 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n",
16139 VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pCtx->cs.Sel, pCtx->rip));
16140#endif
16141
16142 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
16143 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
16144 return VINF_SUCCESS;
16145}
16146
16147
16148/**
16149 * VM-exit exception handler for \#PF (Page-fault exception).
16150 */
16151static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16152{
16153 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16154 PVM pVM = pVCpu->CTX_SUFF(pVM);
16155 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16156 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
16157 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
16158 AssertRCReturn(rc, rc);
16159
16160 if (!pVM->hm.s.fNestedPaging)
16161 { /* likely */ }
16162 else
16163 {
16164#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
16165 Assert(pVCpu->hm.s.fUsingDebugLoop);
16166#endif
16167 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
16168 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
16169 {
16170 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
16171 pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
16172 }
16173 else
16174 {
16175 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
16176 hmR0VmxSetPendingXcptDF(pVCpu);
16177 Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
16178 }
16179 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
16180 return rc;
16181 }
16182
16183 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
16184 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
16185 if (pVmxTransient->fVectoringPF)
16186 {
16187 Assert(pVCpu->hm.s.Event.fPending);
16188 return VINF_EM_RAW_INJECT_TRPM_EVENT;
16189 }
16190
16191 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16192 rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16193 AssertRCReturn(rc, rc);
16194
16195 Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
16196 pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
16197
16198 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
16199 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
16200
16201 Log4Func(("#PF: rc=%Rrc\n", rc));
16202 if (rc == VINF_SUCCESS)
16203 {
16204 /*
16205 * This is typically a shadow page table sync or a MMIO instruction. But we may have
16206 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
16207 */
16208 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16209 TRPMResetTrap(pVCpu);
16210 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
16211 return rc;
16212 }
16213
16214 if (rc == VINF_EM_RAW_GUEST_TRAP)
16215 {
16216 if (!pVmxTransient->fVectoringDoublePF)
16217 {
16218 /* It's a guest page fault and needs to be reflected to the guest. */
16219 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
16220 TRPMResetTrap(pVCpu);
16221 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
16222 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
16223 uGstErrorCode, pVmxTransient->uExitQual);
16224 }
16225 else
16226 {
16227 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
16228 TRPMResetTrap(pVCpu);
16229 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
16230 hmR0VmxSetPendingXcptDF(pVCpu);
16231 Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
16232 }
16233
16234 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
16235 return VINF_SUCCESS;
16236 }
16237
16238 TRPMResetTrap(pVCpu);
16239 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
16240 return rc;
16241}
16242
16243
16244/**
16245 * VM-exit helper for LMSW.
16246 */
16247static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst)
16248{
16249 int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
16250 AssertRCReturn(rc, rc);
16251
16252 VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
16253 AssertMsg( rcStrict == VINF_SUCCESS
16254 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
16255
16256 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
16257 if (rcStrict == VINF_IEM_RAISED_XCPT)
16258 {
16259 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16260 rcStrict = VINF_SUCCESS;
16261 }
16262
16263 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
16264 Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
16265 return rcStrict;
16266}
16267
16268
16269/**
16270 * VM-exit helper for CLTS.
16271 */
16272static VBOXSTRICTRC hmR0VmxExitClts(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr)
16273{
16274 int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
16275 AssertRCReturn(rc, rc);
16276
16277 VBOXSTRICTRC rcStrict = IEMExecDecodedClts(pVCpu, cbInstr);
16278 AssertMsg( rcStrict == VINF_SUCCESS
16279 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
16280
16281 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
16282 if (rcStrict == VINF_IEM_RAISED_XCPT)
16283 {
16284 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16285 rcStrict = VINF_SUCCESS;
16286 }
16287
16288 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
16289 Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
16290 return rcStrict;
16291}
16292
16293
16294/**
16295 * VM-exit helper for MOV from CRx (CRx read).
16296 */
16297static VBOXSTRICTRC hmR0VmxExitMovFromCrX(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
16298{
16299 Assert(iCrReg < 16);
16300 Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
16301
16302 int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
16303 AssertRCReturn(rc, rc);
16304
16305 VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
16306 AssertMsg( rcStrict == VINF_SUCCESS
16307 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
16308
16309 if (iGReg == X86_GREG_xSP)
16310 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RSP);
16311 else
16312 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
16313#ifdef VBOX_WITH_STATISTICS
16314 switch (iCrReg)
16315 {
16316 case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
16317 case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
16318 case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
16319 case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
16320 case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
16321 }
16322#endif
16323 Log4Func(("CR%d Read access rcStrict=%Rrc\n", iCrReg, VBOXSTRICTRC_VAL(rcStrict)));
16324 return rcStrict;
16325}
16326
16327
16328/**
16329 * VM-exit helper for MOV to CRx (CRx write).
16330 */
16331static VBOXSTRICTRC hmR0VmxExitMovToCrX(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
16332{
16333 int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
16334 AssertRCReturn(rc, rc);
16335
16336 VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
16337 AssertMsg( rcStrict == VINF_SUCCESS
16338 || rcStrict == VINF_IEM_RAISED_XCPT
16339 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
16340
16341 switch (iCrReg)
16342 {
16343 case 0:
16344 {
16345 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
16346 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
16347 Log4Func(("CR0 write. rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
16348 break;
16349 }
16350
16351 case 2:
16352 {
16353 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
16354 /* Nothing to do here, CR2 it's not part of the VMCS. */
16355 break;
16356 }
16357
16358 case 3:
16359 {
16360 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR3);
16361 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
16362 Log4Func(("CR3 write. rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
16363 break;
16364 }
16365
16366 case 4:
16367 {
16368 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR4);
16369 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
16370 Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
16371 pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
16372 break;
16373 }
16374
16375 case 8:
16376 {
16377 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
16378 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_APIC_TPR);
16379 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
16380 break;
16381 }
16382
16383 default:
16384 AssertMsgFailed(("Invalid CRx register %#x\n", iCrReg));
16385 break;
16386 }
16387
16388 if (rcStrict == VINF_IEM_RAISED_XCPT)
16389 {
16390 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16391 rcStrict = VINF_SUCCESS;
16392 }
16393 return rcStrict;
16394}
16395
16396
16397/**
16398 * VM-exit helper for handling host NMIs.
16399 */
16400static VBOXSTRICTRC hmR0VmxExitHostNmi(PVMCPU pVCpu)
16401{
16402 VMXDispatchHostNmi();
16403
16404 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
16405 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
16406 return VINF_SUCCESS;
16407}
16408
16409
16410#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
16411/** @name VMX instruction handlers.
16412 * @{
16413 */
16414/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16415/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VMX instructions VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16416/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16417
16418/**
16419 * VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
16420 */
16421HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16422{
16423 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16424
16425 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16426 rc |= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
16427 | CPUMCTX_EXTRN_HWVIRT
16428 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16429 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16430 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16431 AssertRCReturn(rc, rc);
16432
16433 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16434
16435 VMXVEXITINFO ExitInfo;
16436 RT_ZERO(ExitInfo);
16437 ExitInfo.uReason = pVmxTransient->uExitReason;
16438 ExitInfo.u64Qual = pVmxTransient->uExitQual;
16439 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16440 ExitInfo.cbInstr = pVmxTransient->cbInstr;
16441 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16442
16443 VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
16444 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16445 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
16446 else if (rcStrict == VINF_IEM_RAISED_XCPT)
16447 {
16448 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16449 rcStrict = VINF_SUCCESS;
16450 }
16451 return rcStrict;
16452}
16453
16454
16455/**
16456 * VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
16457 */
16458HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16459{
16460 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16461
16462 /* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
16463 otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
16464 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16465 rc |= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16466 AssertRCReturn(rc, rc);
16467
16468 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16469
16470 VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMLAUNCH);
16471 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16472 {
16473 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16474 if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
16475 rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
16476 }
16477 Assert(rcStrict != VINF_IEM_RAISED_XCPT);
16478 return rcStrict;
16479}
16480
16481
16482/**
16483 * VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
16484 */
16485HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16486{
16487 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16488
16489 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16490 rc |= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
16491 | CPUMCTX_EXTRN_HWVIRT
16492 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16493 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16494 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16495 AssertRCReturn(rc, rc);
16496
16497 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16498
16499 VMXVEXITINFO ExitInfo;
16500 RT_ZERO(ExitInfo);
16501 ExitInfo.uReason = pVmxTransient->uExitReason;
16502 ExitInfo.u64Qual = pVmxTransient->uExitQual;
16503 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16504 ExitInfo.cbInstr = pVmxTransient->cbInstr;
16505 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16506
16507 VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
16508 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16509 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
16510 else if (rcStrict == VINF_IEM_RAISED_XCPT)
16511 {
16512 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16513 rcStrict = VINF_SUCCESS;
16514 }
16515 return rcStrict;
16516}
16517
16518
16519/**
16520 * VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
16521 */
16522HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16523{
16524 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16525
16526 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16527 rc |= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
16528 | CPUMCTX_EXTRN_HWVIRT
16529 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16530 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16531 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16532 AssertRCReturn(rc, rc);
16533
16534 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16535
16536 VMXVEXITINFO ExitInfo;
16537 RT_ZERO(ExitInfo);
16538 ExitInfo.uReason = pVmxTransient->uExitReason;
16539 ExitInfo.u64Qual = pVmxTransient->uExitQual;
16540 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16541 ExitInfo.cbInstr = pVmxTransient->cbInstr;
16542 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
16543
16544 VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
16545 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16546 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
16547 else if (rcStrict == VINF_IEM_RAISED_XCPT)
16548 {
16549 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16550 rcStrict = VINF_SUCCESS;
16551 }
16552 return rcStrict;
16553}
16554
16555
16556/**
16557 * VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Conditional VM-exit.
16558 */
16559HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16560{
16561 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16562
16563 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16564 rc |= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
16565 | CPUMCTX_EXTRN_HWVIRT
16566 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16567 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16568 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16569 AssertRCReturn(rc, rc);
16570
16571 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16572
16573 VMXVEXITINFO ExitInfo;
16574 RT_ZERO(ExitInfo);
16575 ExitInfo.uReason = pVmxTransient->uExitReason;
16576 ExitInfo.u64Qual = pVmxTransient->uExitQual;
16577 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16578 ExitInfo.cbInstr = pVmxTransient->cbInstr;
16579 if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16580 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
16581
16582 VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
16583 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16584 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
16585 else if (rcStrict == VINF_IEM_RAISED_XCPT)
16586 {
16587 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16588 rcStrict = VINF_SUCCESS;
16589 }
16590 return rcStrict;
16591}
16592
16593
16594/**
16595 * VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
16596 */
16597HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16598{
16599 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16600
16601 /* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
16602 otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
16603 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16604 rc |= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16605 AssertRCReturn(rc, rc);
16606
16607 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16608
16609 VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMRESUME);
16610 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16611 {
16612 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16613 if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
16614 rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
16615 }
16616 Assert(rcStrict != VINF_IEM_RAISED_XCPT);
16617 return rcStrict;
16618}
16619
16620
16621/**
16622 * VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Conditional VM-exit.
16623 */
16624HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16625{
16626 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16627
16628 /*
16629 * Although we should not get VMWRITE VM-exits for shadow VMCS fields, since
16630 * our HM hook that gets invoked when IEM's VMWRITE instruction emulation
16631 * modifies the current VMCS signals re-loading the entire shadow VMCS, we
16632 * should also save the entire shadow VMCS here.
16633 */
16634 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16635 rc |= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
16636 | CPUMCTX_EXTRN_HWVIRT
16637 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16638 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16639 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16640 AssertRCReturn(rc, rc);
16641
16642 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16643
16644 VMXVEXITINFO ExitInfo;
16645 RT_ZERO(ExitInfo);
16646 ExitInfo.uReason = pVmxTransient->uExitReason;
16647 ExitInfo.u64Qual = pVmxTransient->uExitQual;
16648 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16649 ExitInfo.cbInstr = pVmxTransient->cbInstr;
16650 if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16651 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16652
16653 /** @todo NSTVMX: Remove later. */
16654 Log4Func(("VMWRITE: %#x\n", pVCpu->cpum.GstCtx.aGRegs[ExitInfo.InstrInfo.VmreadVmwrite.iReg2].u32));
16655
16656 VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
16657 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16658 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
16659 else if (rcStrict == VINF_IEM_RAISED_XCPT)
16660 {
16661 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16662 rcStrict = VINF_SUCCESS;
16663 }
16664 return rcStrict;
16665}
16666
16667
16668/**
16669 * VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
16670 */
16671HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16672{
16673 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16674
16675 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16676 rc |= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
16677 | CPUMCTX_EXTRN_HWVIRT
16678 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
16679 AssertRCReturn(rc, rc);
16680
16681 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16682
16683 VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
16684 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16685 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_HWVIRT);
16686 else if (rcStrict == VINF_IEM_RAISED_XCPT)
16687 {
16688 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16689 rcStrict = VINF_SUCCESS;
16690 }
16691 return rcStrict;
16692}
16693
16694
16695/**
16696 * VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
16697 */
16698HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16699{
16700 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16701
16702 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16703 rc |= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
16704 | CPUMCTX_EXTRN_HWVIRT
16705 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16706 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16707 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16708 AssertRCReturn(rc, rc);
16709
16710 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16711
16712 VMXVEXITINFO ExitInfo;
16713 RT_ZERO(ExitInfo);
16714 ExitInfo.uReason = pVmxTransient->uExitReason;
16715 ExitInfo.u64Qual = pVmxTransient->uExitQual;
16716 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16717 ExitInfo.cbInstr = pVmxTransient->cbInstr;
16718 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16719
16720 VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
16721 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16722 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
16723 else if (rcStrict == VINF_IEM_RAISED_XCPT)
16724 {
16725 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16726 rcStrict = VINF_SUCCESS;
16727 }
16728 return rcStrict;
16729}
16730
16731
16732/**
16733 * VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
16734 */
16735HMVMX_EXIT_DECL hmR0VmxExitInvvpid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16736{
16737 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16738
16739 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16740 rc |= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
16741 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16742 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16743 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16744 AssertRCReturn(rc, rc);
16745
16746 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16747
16748 VMXVEXITINFO ExitInfo;
16749 RT_ZERO(ExitInfo);
16750 ExitInfo.uReason = pVmxTransient->uExitReason;
16751 ExitInfo.u64Qual = pVmxTransient->uExitQual;
16752 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16753 ExitInfo.cbInstr = pVmxTransient->cbInstr;
16754 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16755
16756 VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
16757 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16758 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
16759 else if (rcStrict == VINF_IEM_RAISED_XCPT)
16760 {
16761 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16762 rcStrict = VINF_SUCCESS;
16763 }
16764 return rcStrict;
16765}
16766
16767/** @} */
16768
16769/** @name Nested-guest VM-exit handlers.
16770 * @{
16771 */
16772/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16773/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- Nested-guest VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16774/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16775
16776/**
16777 * Nested-guest VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
16778 * Conditional VM-exit.
16779 */
16780HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmiNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16781{
16782 HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16783
16784 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
16785 AssertRCReturn(rc, rc);
16786
16787 uint64_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
16788 Assert(VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo));
16789 uint32_t const uExtIntType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
16790
16791 /*
16792 * Make sure not to use stale/previous VM-exit instruction length since we read the
16793 * instruction length from the VMCS below only for software exceptions and privileged
16794 * software exceptions but we pass it for all exception VM-exits below.
16795 */
16796 pVmxTransient->cbInstr = 0;
16797
16798 switch (uExtIntType)
16799 {
16800 /*
16801 * Physical NMIs:
16802 * We shouldn't direct host physical NMIs to the nested-guest. Dispatch it to the
16803 * host.
16804 */
16805 case VMX_EXIT_INT_INFO_TYPE_NMI:
16806 return hmR0VmxExitHostNmi(pVCpu);
16807
16808 /*
16809 * Hardware exceptions,
16810 * Software exceptions,
16811 * Privileged software exceptions:
16812 * Figure out if the exception must be delivered to the guest or the nested-guest.
16813 *
16814 * For VM-exits due to software exceptions (those generated by INT3 or INTO) and privileged
16815 * software exceptions (those generated by INT1/ICEBP) we need to supply the VM-exit instruction
16816 * length.
16817 */
16818 case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
16819 case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
16820 {
16821 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16822 RT_FALL_THRU();
16823 }
16824 case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
16825 {
16826 rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
16827 AssertRCReturn(rc, rc);
16828
16829 uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
16830 bool const fIntercept = CPUMIsGuestVmxXcptInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uVector,
16831 pVmxTransient->uExitIntErrorCode);
16832 if (fIntercept)
16833 {
16834 rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16835 rc |= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16836 rc |= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16837 AssertRCReturn(rc, rc);
16838
16839 VMXVEXITINFO ExitInfo;
16840 RT_ZERO(ExitInfo);
16841 ExitInfo.cbInstr = pVmxTransient->cbInstr;
16842 ExitInfo.u64Qual = pVmxTransient->uExitQual;
16843
16844 VMXVEXITEVENTINFO ExitEventInfo;
16845 RT_ZERO(ExitEventInfo);
16846 ExitEventInfo.uExitIntInfo = pVmxTransient->uExitIntInfo;
16847 ExitEventInfo.uExitIntErrCode = pVmxTransient->uExitIntErrorCode;
16848 ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16849 ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16850
16851 return IEMExecVmxVmexitXcpt(pVCpu, &ExitInfo, &ExitEventInfo);
16852 }
16853
16854 /* Nested paging is currently a requirement, otherwise we would need to handle shadow #PFs. */
16855 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
16856
16857 /* If the guest hypervisor is not intercepting the exception, forward it to the guest. */
16858 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo), pVmxTransient->cbInstr,
16859 pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
16860 return VINF_SUCCESS;
16861 }
16862
16863 /*
16864 * Software interrupts:
16865 * VM-exits cannot be caused by software interrupts.
16866 *
16867 * External interrupts:
16868 * This should only happen when "acknowledge external interrupts on VM-exit"
16869 * control is set. However, we don't set it when executing guests or
16870 * nested-guests. For nested-guests it is emulated while injecting interrupts into
16871 * the guest.
16872 */
16873 case VMX_EXIT_INT_INFO_TYPE_SW_INT:
16874 case VMX_EXIT_INT_INFO_TYPE_EXT_INT:
16875 default:
16876 {
16877 pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
16878 return VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
16879 }
16880 }
16881}
16882
16883
16884/**
16885 * Nested-guest VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT).
16886 * Unconditional VM-exit.
16887 */
16888HMVMX_EXIT_DECL hmR0VmxExitTripleFaultNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16889{
16890 HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16891 return IEMExecVmxVmexitTripleFault(pVCpu);
16892}
16893
16894
16895/**
16896 * Nested-guest VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
16897 */
16898HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindowNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16899{
16900 HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16901
16902 if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT))
16903 return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16904 return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16905}
16906
16907
16908/**
16909 * Nested-guest VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
16910 */
16911HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindowNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16912{
16913 HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16914
16915 if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT))
16916 return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16917 return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16918}
16919
16920
16921/**
16922 * Nested-guest VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH).
16923 * Unconditional VM-exit.
16924 */
16925HMVMX_EXIT_DECL hmR0VmxExitTaskSwitchNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16926{
16927 HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16928
16929 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16930 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16931 rc |= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16932 rc |= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16933 AssertRCReturn(rc, rc);
16934
16935 VMXVEXITINFO ExitInfo;
16936 RT_ZERO(ExitInfo);
16937 ExitInfo.cbInstr = pVmxTransient->cbInstr;
16938 ExitInfo.u64Qual = pVmxTransient->uExitQual;
16939
16940 VMXVEXITEVENTINFO ExitEventInfo;
16941 RT_ZERO(ExitInfo);
16942 ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16943 ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16944 return IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
16945}
16946
16947
16948/**
16949 * Nested-guest VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
16950 */
16951HMVMX_EXIT_DECL hmR0VmxExitHltNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16952{
16953 HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16954
16955 if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
16956 {
16957 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16958 AssertRCReturn(rc, rc);
16959 return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16960 }
16961 return hmR0VmxExitHlt(pVCpu, pVmxTransient);
16962}
16963
16964
16965/**
16966 * Nested-guest VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
16967 */
16968HMVMX_EXIT_DECL hmR0VmxExitInvlpgNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16969{
16970 HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16971
16972 if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
16973 {
16974 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16975 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16976 AssertRCReturn(rc, rc);
16977
16978 VMXVEXITINFO ExitInfo;
16979 RT_ZERO(ExitInfo);
16980 ExitInfo.uReason = pVmxTransient->uExitReason;
16981 ExitInfo.cbInstr = pVmxTransient->cbInstr;
16982 ExitInfo.u64Qual = pVmxTransient->uExitQual;
16983 return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16984 }
16985 return hmR0VmxExitInvlpg(pVCpu, pVmxTransient);
16986}
16987
16988
16989/**
16990 * Nested-guest VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
16991 */
16992HMVMX_EXIT_DECL hmR0VmxExitRdpmcNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16993{
16994 HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16995
16996 if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDPMC_EXIT))
16997 {
16998 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16999 AssertRCReturn(rc, rc);
17000 return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17001 }
17002 return hmR0VmxExitRdpmc(pVCpu, pVmxTransient);
17003}
17004
17005
17006/**
17007 * Nested-guest VM-exit handler for VMREAD (VMX_EXIT_VMREAD) and VMWRITE
17008 * (VMX_EXIT_VMWRITE). Conditional VM-exit.
17009 */
17010HMVMX_EXIT_DECL hmR0VmxExitVmreadVmwriteNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17011{
17012 HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17013
17014 Assert( pVmxTransient->uExitReason == VMX_EXIT_VMREAD
17015 || pVmxTransient->uExitReason == VMX_EXIT_VMWRITE);
17016
17017 int rc = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17018 AssertRCReturn(rc, rc);
17019
17020 uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
17021 Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
17022 uint64_t u64FieldEnc = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
17023
17024 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
17025 if (!CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
17026 u64FieldEnc &= UINT64_C(0xffffffff);
17027
17028 if (CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, pVmxTransient->uExitReason, u64FieldEnc))
17029 {
17030 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17031 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17032 AssertRCReturn(rc, rc);
17033
17034 VMXVEXITINFO ExitInfo;
17035 RT_ZERO(ExitInfo);
17036 ExitInfo.uReason = pVmxTransient->uExitReason;
17037 ExitInfo.cbInstr = pVmxTransient->cbInstr;
17038 ExitInfo.u64Qual = pVmxTransient->uExitQual;
17039 ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17040 return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17041 }
17042
17043 if (pVmxTransient->uExitReason == VMX_EXIT_VMREAD)
17044 return hmR0VmxExitVmread(pVCpu, pVmxTransient);
17045 return hmR0VmxExitVmwrite(pVCpu, pVmxTransient);
17046}
17047
17048
17049/**
17050 * Nested-guest VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
17051 */
17052HMVMX_EXIT_DECL hmR0VmxExitRdtscNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17053{
17054 HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17055
17056 if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
17057 {
17058 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17059 AssertRCReturn(rc, rc);
17060 return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17061 }
17062
17063 return hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
17064}
17065
17066
17067/**
17068 * Nested-guest VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX).
17069 * Conditional VM-exit.
17070 */
17071HMVMX_EXIT_DECL hmR0VmxExitMovCRxNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17072{
17073 HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17074
17075 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17076 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17077 AssertRCReturn(rc, rc);
17078
17079 VBOXSTRICTRC rcStrict;
17080 uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
17081 switch (uAccessType)
17082 {
17083 case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
17084 {
17085 uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
17086 uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
17087 Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
17088 uint64_t const uNewCrX = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
17089 if (CPUMIsGuestVmxMovToCr0Cr4InterceptSet(pVCpu, &pVCpu->cpum.GstCtx, iCrReg, uNewCrX))
17090 {
17091 VMXVEXITINFO ExitInfo;
17092 RT_ZERO(ExitInfo);
17093 ExitInfo.uReason = pVmxTransient->uExitReason;
17094 ExitInfo.cbInstr = pVmxTransient->cbInstr;
17095 ExitInfo.u64Qual = pVmxTransient->uExitQual;
17096 rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17097 }
17098 else
17099 rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
17100 break;
17101 }
17102
17103 case VMX_EXIT_QUAL_CRX_ACCESS_READ:
17104 {
17105 /*
17106 * CR0/CR4 reads do not cause VM-exits, the read-shadow is used (subject to masking).
17107 * CR2 reads do not cause a VM-exit.
17108 * CR3 reads cause a VM-exit depending on the "CR3 store exiting" control.
17109 * CR8 reads cause a VM-exit depending on the "CR8 store exiting" control.
17110 */
17111 uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
17112 if ( iCrReg == 3
17113 || iCrReg == 8)
17114 {
17115 static const uint32_t s_aCrXReadIntercepts[] = { 0, 0, 0, VMX_PROC_CTLS_CR3_STORE_EXIT, 0,
17116 0, 0, 0, VMX_PROC_CTLS_CR8_STORE_EXIT };
17117 uint32_t const uIntercept = s_aCrXReadIntercepts[iCrReg];
17118 if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uIntercept))
17119 {
17120 VMXVEXITINFO ExitInfo;
17121 RT_ZERO(ExitInfo);
17122 ExitInfo.uReason = pVmxTransient->uExitReason;
17123 ExitInfo.cbInstr = pVmxTransient->cbInstr;
17124 ExitInfo.u64Qual = pVmxTransient->uExitQual;
17125 rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17126 }
17127 else
17128 {
17129 uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
17130 rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
17131 }
17132 }
17133 else
17134 {
17135 AssertMsgFailed(("MOV from CR%d VM-exit must not happen\n", iCrReg));
17136 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, iCrReg);
17137 }
17138 break;
17139 }
17140
17141 case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
17142 {
17143 PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
17144 Assert(pVmcsNstGst);
17145 uint64_t const uGstHostMask = pVmcsNstGst->u64Cr0Mask.u;
17146 uint64_t const uReadShadow = pVmcsNstGst->u64Cr0ReadShadow.u;
17147 if ( (uGstHostMask & X86_CR0_TS)
17148 && (uReadShadow & X86_CR0_TS))
17149 {
17150 VMXVEXITINFO ExitInfo;
17151 RT_ZERO(ExitInfo);
17152 ExitInfo.uReason = pVmxTransient->uExitReason;
17153 ExitInfo.cbInstr = pVmxTransient->cbInstr;
17154 ExitInfo.u64Qual = pVmxTransient->uExitQual;
17155 rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17156 }
17157 else
17158 rcStrict = hmR0VmxExitClts(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr);
17159 break;
17160 }
17161
17162 case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
17163 {
17164 RTGCPTR GCPtrEffDst;
17165 uint16_t const uNewMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
17166 bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual);
17167 if (fMemOperand)
17168 {
17169 rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
17170 AssertRCReturn(rc, rc);
17171 GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
17172 }
17173 else
17174 GCPtrEffDst = NIL_RTGCPTR;
17175
17176 if (CPUMIsGuestVmxLmswInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uNewMsw))
17177 {
17178 VMXVEXITINFO ExitInfo;
17179 RT_ZERO(ExitInfo);
17180 ExitInfo.uReason = pVmxTransient->uExitReason;
17181 ExitInfo.cbInstr = pVmxTransient->cbInstr;
17182 ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
17183 ExitInfo.u64Qual = pVmxTransient->uExitQual;
17184 rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17185 }
17186 else
17187 rcStrict = hmR0VmxExitLmsw(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, uNewMsw, GCPtrEffDst);
17188 break;
17189 }
17190
17191 default:
17192 {
17193 AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
17194 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
17195 }
17196 }
17197
17198 if (rcStrict == VINF_IEM_RAISED_XCPT)
17199 {
17200 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
17201 rcStrict = VINF_SUCCESS;
17202 }
17203 return rcStrict;
17204}
17205
17206
17207/**
17208 * Nested-guest VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX).
17209 * Conditional VM-exit.
17210 */
17211HMVMX_EXIT_DECL hmR0VmxExitMovDRxNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17212{
17213 HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17214
17215 if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
17216 {
17217 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17218 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17219 AssertRCReturn(rc, rc);
17220
17221 VMXVEXITINFO ExitInfo;
17222 RT_ZERO(ExitInfo);
17223 ExitInfo.cbInstr = pVmxTransient->cbInstr;
17224 ExitInfo.u64Qual = pVmxTransient->uExitQual;
17225 return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17226 }
17227 return hmR0VmxExitMovDRx(pVCpu, pVmxTransient);
17228}
17229
17230
17231/**
17232 * Nested-guest VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR).
17233 * Conditional VM-exit.
17234 */
17235HMVMX_EXIT_DECL hmR0VmxExitIoInstrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17236{
17237 HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17238
17239 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17240 AssertRCReturn(rc, rc);
17241
17242 uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
17243 uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
17244 AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
17245
17246 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
17247 uint8_t const cbAccess = s_aIOSizes[uIOSize];
17248 if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
17249 {
17250 /*
17251 * IN/OUT instruction:
17252 * - Provides VM-exit instruction length.
17253 *
17254 * INS/OUTS instruction:
17255 * - Provides VM-exit instruction length.
17256 * - Provides Guest-linear address.
17257 * - Optionally provides VM-exit instruction info (depends on CPU feature).
17258 */
17259 PVM pVM = pVCpu->CTX_SUFF(pVM);
17260 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17261 AssertRCReturn(rc, rc);
17262
17263 /* Make sure we don't use stale/uninitialized VMX-transient info. below. */
17264 pVmxTransient->ExitInstrInfo.u = 0;
17265 pVmxTransient->uGuestLinearAddr = 0;
17266
17267 bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
17268 bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
17269 if (fIOString)
17270 {
17271 rc |= hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
17272 if (fVmxInsOutsInfo)
17273 {
17274 Assert(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
17275 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17276 }
17277 }
17278 AssertRCReturn(rc, rc);
17279
17280 VMXVEXITINFO ExitInfo;
17281 RT_ZERO(ExitInfo);
17282 ExitInfo.uReason = pVmxTransient->uExitReason;
17283 ExitInfo.cbInstr = pVmxTransient->cbInstr;
17284 ExitInfo.u64Qual = pVmxTransient->uExitQual;
17285 ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17286 ExitInfo.u64GuestLinearAddr = pVmxTransient->uGuestLinearAddr;
17287 return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17288 }
17289 return hmR0VmxExitIoInstr(pVCpu, pVmxTransient);
17290}
17291
17292
17293/**
17294 * Nested-guest VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
17295 */
17296HMVMX_EXIT_DECL hmR0VmxExitRdmsrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17297{
17298 HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17299
17300 uint32_t fMsrpm;
17301 if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
17302 fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
17303 else
17304 fMsrpm = VMXMSRPM_EXIT_RD;
17305
17306 if (fMsrpm & VMXMSRPM_EXIT_RD)
17307 {
17308 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17309 AssertRCReturn(rc, rc);
17310 return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17311 }
17312 return hmR0VmxExitRdmsr(pVCpu, pVmxTransient);
17313}
17314
17315
17316/**
17317 * Nested-guest VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
17318 */
17319HMVMX_EXIT_DECL hmR0VmxExitWrmsrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17320{
17321 HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17322
17323 uint32_t fMsrpm;
17324 if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
17325 fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
17326 else
17327 fMsrpm = VMXMSRPM_EXIT_WR;
17328
17329 if (fMsrpm & VMXMSRPM_EXIT_WR)
17330 {
17331 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17332 AssertRCReturn(rc, rc);
17333 return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17334 }
17335 return hmR0VmxExitWrmsr(pVCpu, pVmxTransient);
17336}
17337
17338
17339/**
17340 * Nested-guest VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
17341 */
17342HMVMX_EXIT_DECL hmR0VmxExitMwaitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17343{
17344 HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17345
17346 if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
17347 {
17348 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17349 AssertRCReturn(rc, rc);
17350 return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17351 }
17352 return hmR0VmxExitMwait(pVCpu, pVmxTransient);
17353}
17354
17355
17356/**
17357 * Nested-guest VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional
17358 * VM-exit.
17359 */
17360HMVMX_EXIT_DECL hmR0VmxExitMtfNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17361{
17362 HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17363
17364 /** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
17365 return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
17366}
17367
17368
17369/**
17370 * Nested-guest VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
17371 */
17372HMVMX_EXIT_DECL hmR0VmxExitMonitorNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17373{
17374 HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17375
17376 if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
17377 {
17378 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17379 AssertRCReturn(rc, rc);
17380 return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17381 }
17382 return hmR0VmxExitMonitor(pVCpu, pVmxTransient);
17383}
17384
17385
17386/**
17387 * Nested-guest VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
17388 */
17389HMVMX_EXIT_DECL hmR0VmxExitPauseNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17390{
17391 HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17392
17393 /** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
17394 * PAUSE when executing a nested-guest? If it does not, we would not need
17395 * to check for the intercepts here. Just call VM-exit... */
17396
17397 /* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
17398 if ( CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
17399 || CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
17400 {
17401 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17402 AssertRCReturn(rc, rc);
17403 return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17404 }
17405 return hmR0VmxExitPause(pVCpu, pVmxTransient);
17406}
17407
17408
17409/**
17410 * Nested-guest VM-exit handler for when the TPR value is lowered below the
17411 * specified threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
17412 */
17413HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThresholdNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17414{
17415 HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17416
17417 if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_TPR_SHADOW))
17418 return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
17419 return hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient);
17420}
17421
17422
17423/**
17424 * Nested-guest VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional
17425 * VM-exit.
17426 */
17427HMVMX_EXIT_DECL hmR0VmxExitApicAccessNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17428{
17429 HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17430
17431 Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
17432 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17433 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17434 rc |= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
17435 rc |= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
17436 AssertRCReturn(rc, rc);
17437
17438 VMXVEXITINFO ExitInfo;
17439 RT_ZERO(ExitInfo);
17440 ExitInfo.cbInstr = pVmxTransient->cbInstr;
17441 ExitInfo.u64Qual = pVmxTransient->uExitQual;
17442
17443 VMXVEXITEVENTINFO ExitEventInfo;
17444 RT_ZERO(ExitInfo);
17445 ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
17446 ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
17447 return IEMExecVmxVmexitApicAccess(pVCpu, &ExitInfo, &ExitEventInfo);
17448}
17449
17450
17451/**
17452 * Nested-guest VM-exit handler for APIC write emulation (VMX_EXIT_APIC_WRITE).
17453 * Conditional VM-exit.
17454 */
17455HMVMX_EXIT_DECL hmR0VmxExitApicWriteNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17456{
17457 HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17458
17459 Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_APIC_REG_VIRT));
17460 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17461 AssertRCReturn(rc, rc);
17462
17463 return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
17464}
17465
17466
17467/**
17468 * Nested-guest VM-exit handler for virtualized EOI (VMX_EXIT_VIRTUALIZED_EOI).
17469 * Conditional VM-exit.
17470 */
17471HMVMX_EXIT_DECL hmR0VmxExitVirtEoiNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17472{
17473 HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17474
17475 Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_INT_DELIVERY));
17476 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17477 AssertRCReturn(rc, rc);
17478
17479 return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
17480}
17481
17482
17483/**
17484 * Nested-guest VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
17485 */
17486HMVMX_EXIT_DECL hmR0VmxExitRdtscpNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17487{
17488 HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17489
17490 if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
17491 {
17492 Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_RDTSCP));
17493 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17494 AssertRCReturn(rc, rc);
17495 return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17496 }
17497 return hmR0VmxExitRdtscp(pVCpu, pVmxTransient);
17498}
17499
17500
17501/**
17502 * Nested-guest VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
17503 */
17504HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvdNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17505{
17506 HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17507
17508 if (CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
17509 {
17510 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17511 AssertRCReturn(rc, rc);
17512 return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17513 }
17514 return hmR0VmxExitWbinvd(pVCpu, pVmxTransient);
17515}
17516
17517
17518/**
17519 * Nested-guest VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
17520 */
17521HMVMX_EXIT_DECL hmR0VmxExitInvpcidNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17522{
17523 HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17524
17525 if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
17526 {
17527 Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_INVPCID));
17528 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17529 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17530 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17531 AssertRCReturn(rc, rc);
17532
17533 VMXVEXITINFO ExitInfo;
17534 RT_ZERO(ExitInfo);
17535 ExitInfo.uReason = pVmxTransient->uExitReason;
17536 ExitInfo.cbInstr = pVmxTransient->cbInstr;
17537 ExitInfo.u64Qual = pVmxTransient->uExitQual;
17538 ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17539 return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17540 }
17541 return hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
17542}
17543
17544
17545/**
17546 * Nested-guest VM-exit handler for invalid-guest state
17547 * (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error VM-exit.
17548 */
17549HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestStateNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17550{
17551 HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17552
17553 /*
17554 * Currently this should never happen because we fully emulate VMLAUNCH/VMRESUME in IEM.
17555 * So if it does happen, it indicates a bug possibly in the hardware-assisted VMX code.
17556 * Handle it like it's in an invalid guest state of the outer guest.
17557 *
17558 * When the fast path is implemented, this should be changed to cause the corresponding
17559 * nested-guest VM-exit.
17560 */
17561 return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
17562}
17563
17564
17565/**
17566 * Nested-guest VM-exit handler for instructions that cause VM-exits uncondtionally
17567 * and only provide the instruction length.
17568 *
17569 * Unconditional VM-exit.
17570 */
17571HMVMX_EXIT_DECL hmR0VmxExitInstrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17572{
17573 HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17574
17575#ifdef VBOX_STRICT
17576 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17577 switch (pVmxTransient->uExitReason)
17578 {
17579 case VMX_EXIT_ENCLS:
17580 Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_ENCLS_EXIT));
17581 break;
17582
17583 case VMX_EXIT_VMFUNC:
17584 Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_VMFUNC));
17585 break;
17586 }
17587#endif
17588
17589 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17590 AssertRCReturn(rc, rc);
17591 return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17592}
17593
17594
17595/**
17596 * Nested-guest VM-exit handler for instructions that provide instruction length as
17597 * well as more information.
17598 *
17599 * Unconditional VM-exit.
17600 */
17601HMVMX_EXIT_DECL hmR0VmxExitInstrWithInfoNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17602{
17603 HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17604
17605#ifdef VBOX_STRICT
17606 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17607 switch (pVmxTransient->uExitReason)
17608 {
17609 case VMX_EXIT_GDTR_IDTR_ACCESS:
17610 case VMX_EXIT_LDTR_TR_ACCESS:
17611 Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT));
17612 break;
17613
17614 case VMX_EXIT_RDRAND:
17615 Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDRAND_EXIT));
17616 break;
17617
17618 case VMX_EXIT_RDSEED:
17619 Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDSEED_EXIT));
17620 break;
17621
17622 case VMX_EXIT_XSAVES:
17623 case VMX_EXIT_XRSTORS:
17624 /** @todo NSTVMX: Verify XSS-bitmap. */
17625 Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_XSAVES_XRSTORS));
17626 break;
17627
17628 case VMX_EXIT_UMWAIT:
17629 case VMX_EXIT_TPAUSE:
17630 Assert(CPUMIsGuestVmxProcCtlsSet(pVCpu, pCtx, VMX_PROC_CTLS_RDTSC_EXIT));
17631 Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_USER_WAIT_PAUSE));
17632 break;
17633 }
17634#endif
17635
17636 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17637 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17638 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17639 AssertRCReturn(rc, rc);
17640
17641 VMXVEXITINFO ExitInfo;
17642 RT_ZERO(ExitInfo);
17643 ExitInfo.uReason = pVmxTransient->uExitReason;
17644 ExitInfo.cbInstr = pVmxTransient->cbInstr;
17645 ExitInfo.u64Qual = pVmxTransient->uExitQual;
17646 ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17647 return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17648}
17649
17650/** @} */
17651
17652#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
17653
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette